Skip to content

Commit 9dc287c

Browse files
at055612at055612
authored
Merge pull request #5550 from gchq/gh-5549-jffi-extract-dir
PR for #5549 - jnr ffi is extracting a native lib to /tmp which doesn't work if /tmp is mounted with `noexec`
2 parents 9142dc3 + 9802bdc commit 9dc287c

4 files changed

Lines changed: 200 additions & 36 deletions

File tree

‎stroom-config/stroom-config-app/src/test/resources/stroom/config/app/expected.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -589,6 +589,7 @@ appConfig:
589589
lifecycle:
590590
enabled: true
591591
lmdbLibrary:
592+
providedJffiLibraryPath: null
592593
providedSystemLibraryPath: null
593594
systemLibraryExtractDir: "lmdb_library"
594595
logging:

‎stroom-lmdb/src/main/java/stroom/lmdb/LmdbLibrary.java‎

Lines changed: 91 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import stroom.util.logging.LambdaLogger;
2222
import stroom.util.logging.LambdaLoggerFactory;
2323
import stroom.util.logging.LogUtil;
24+
import stroom.util.shared.NullSafe;
2425

2526
import jakarta.inject.Inject;
2627
import jakarta.inject.Provider;
@@ -31,7 +32,6 @@
3132
import java.nio.file.Files;
3233
import java.nio.file.Path;
3334
import java.util.Objects;
34-
import java.util.Optional;
3535

3636
@Singleton // The LMDB lib is dealt with statically by LMDB java so only want to initialise it once
3737
public class LmdbLibrary {
@@ -67,49 +67,107 @@ private synchronized void configureLibraryUnderLock(final LmdbLibraryConfig lmdb
6767
if (!HAS_LIBRARY_BEEN_CONFIGURED) {
6868
LOGGER.info("Configuring LMDB system library");
6969

70-
final Path lmdbSystemLibraryPath = Optional.ofNullable(lmdbLibraryConfig.getProvidedSystemLibraryPath())
71-
.map(pathCreator::toAppPath)
72-
.orElse(null);
70+
LOGGER.debug(() -> LogUtil.message("""
71+
configureLibraryUnderLock() - lmdbLibraryConfig: {},
72+
{},
73+
{},
74+
{},
75+
{}""",
76+
lmdbLibraryConfig,
77+
dumpSystemProp(LmdbLibraryConfig.LMDB_EXTRACT_DIR_PROP),
78+
dumpSystemProp(LmdbLibraryConfig.LMDB_NATIVE_LIB_PROP),
79+
dumpSystemProp(LmdbLibraryConfig.JFFI_EXTRACT_DIR_PROP),
80+
dumpSystemProp(LmdbLibraryConfig.JFFI_NATIVE_LIB_PROP)));
81+
82+
final Path lmdbSystemLibraryPath = NullSafe.get(
83+
lmdbLibraryConfig.getProvidedSystemLibraryPath(),
84+
pathCreator::toAppPath);
85+
final Path jffiNativeLibraryPath = NullSafe.get(
86+
lmdbLibraryConfig.getProvidedJffiLibraryPath(),
87+
pathCreator::toAppPath);
88+
89+
boolean extractLmdb = false;
90+
boolean extractJffi = false;
7391

7492
if (lmdbSystemLibraryPath != null) {
7593
if (!Files.isReadable(lmdbSystemLibraryPath)) {
76-
throw new RuntimeException("Unable to read LMDB system library at " +
77-
lmdbSystemLibraryPath.toAbsolutePath().normalize());
94+
throw new RuntimeException("Unable to read LMDB system library at " + lmdbSystemLibraryPath);
7895
}
7996
// jakarta.validation should ensure the path is valid if set
8097
final String lmdbNativeLibProp = LmdbLibraryConfig.LMDB_NATIVE_LIB_PROP;
81-
System.setProperty(lmdbNativeLibProp, lmdbSystemLibraryPath.toAbsolutePath().normalize().toString());
82-
LOGGER.info("Using provided LMDB system library file. Setting prop {} to '{}'",
98+
System.setProperty(lmdbNativeLibProp, lmdbSystemLibraryPath.toString());
99+
LOGGER.info("Using provided LMDB native library file. Setting prop {} to '{}'",
83100
lmdbNativeLibProp, lmdbSystemLibraryPath);
84101
} else {
85-
final Path systemLibraryExtractDir = getLibraryExtractDir(lmdbLibraryConfig);
102+
extractLmdb = true;
103+
}
104+
105+
final String jffiNativeLibProp = LmdbLibraryConfig.JFFI_NATIVE_LIB_PROP;
106+
final String jffiPropVal = System.getenv(jffiNativeLibProp);
107+
if (NullSafe.isNonBlankString(jffiPropVal)) {
108+
LOGGER.info("Using provided JFFI native library file. Property {} already set to '{}'",
109+
jffiNativeLibProp, jffiPropVal);
110+
} else if (jffiNativeLibraryPath != null) {
111+
if (!Files.isReadable(jffiNativeLibraryPath)) {
112+
throw new RuntimeException("Unable to read JFFI native library at " + jffiNativeLibraryPath);
113+
}
114+
// jakarta.validation should ensure the path is valid if set
115+
System.setProperty(jffiNativeLibProp, jffiNativeLibraryPath.toString());
116+
LOGGER.info("Using provided JFFI native library file. Setting prop {} to '{}'",
117+
jffiNativeLibProp, lmdbSystemLibraryPath);
118+
} else {
119+
extractJffi = true;
120+
}
86121

87-
// LMDB extracts the lib on boot to a unique temp file and should delete on JVM exit,
122+
LOGGER.debug("configureLibraryUnderLock() - extractLmdb: {}, extractJffi: {}", extractLmdb, extractJffi);
123+
if (extractLmdb || extractJffi) {
124+
final Path systemLibraryExtractDir = getLibraryExtractDir(lmdbLibraryConfig);
125+
// LMDB/JFFI extract the lib on boot to a unique temp file and should delete on JVM exit,
88126
// but just in case clear out any old ones.
89127
cleanUpExtractDir(systemLibraryExtractDir);
90128

129+
// Extract them both to the same place as both will need a non 'noexec' mount.
130+
91131
// Set the location to extract the bundled LMDB binary to
92-
final String lmdbExtractDirProp = LmdbLibraryConfig.LMDB_EXTRACT_DIR_PROP;
93-
System.setProperty(
94-
lmdbExtractDirProp,
95-
systemLibraryExtractDir.toAbsolutePath().normalize().toString());
96-
LOGGER.info("Bundled LMDB system library binary will be extracted. Setting prop {} to '{}'",
97-
lmdbExtractDirProp, systemLibraryExtractDir);
98-
HAS_LIBRARY_BEEN_CONFIGURED = true;
132+
if (extractLmdb) {
133+
final String lmdbExtractDirProp = LmdbLibraryConfig.LMDB_EXTRACT_DIR_PROP;
134+
System.setProperty(lmdbExtractDirProp, systemLibraryExtractDir.toString());
135+
LOGGER.info("Bundled LMDB native library binary will be extracted and used. " +
136+
"Setting system prop '{}' to '{}'",
137+
lmdbExtractDirProp, systemLibraryExtractDir);
138+
}
139+
140+
// Set the location to extract the bundled Jffi binary to
141+
if (extractJffi) {
142+
final String jffiExtractDirProp = LmdbLibraryConfig.JFFI_EXTRACT_DIR_PROP;
143+
if (System.getProperty(jffiExtractDirProp) == null) {
144+
// Allow a jvm arg to override our config
145+
System.setProperty(jffiExtractDirProp, systemLibraryExtractDir.toString());
146+
LOGGER.info("Bundled JFFI native library binary will be extracted and used. " +
147+
"Setting system prop '{}' to '{}'",
148+
jffiExtractDirProp, systemLibraryExtractDir);
149+
}
150+
}
99151
}
152+
HAS_LIBRARY_BEEN_CONFIGURED = true;
100153
} else {
101154
LOGGER.debug("Another thread beat us to it");
102155
}
103156
}
104157

158+
private String dumpSystemProp(final String propName) {
159+
return "'" + propName + "': '" + System.getProperty(propName) + "'";
160+
}
161+
105162
private Path getLibraryExtractDir(final LmdbLibraryConfig lmdbLibraryConfig) {
106163
final String extractDirStr = lmdbLibraryConfig.getSystemLibraryExtractDir();
107164

108165
final String extractDirPropName = lmdbLibraryConfig.getFullPathStr(LmdbLibraryConfig.EXTRACT_DIR_PROP_NAME);
109166

110167
Path extractDir;
111168
if (extractDirStr == null) {
112-
LOGGER.warn("LMDB system library extract dir is not set ({}), falling back to temporary directory {}.",
169+
LOGGER.warn("LMDB system library extract dir is not set ({}), falling back to temporary directory {}. " +
170+
"If the temporary directory is mounted with 'noexec' any use of LMDB will error.",
113171
extractDirPropName,
114172
tempDirProvider.get());
115173

@@ -119,6 +177,7 @@ private Path getLibraryExtractDir(final LmdbLibraryConfig lmdbLibraryConfig) {
119177
} else {
120178
extractDir = pathCreator.toAppPath(extractDirStr);
121179
}
180+
extractDir = extractDir.toAbsolutePath().normalize();
122181

123182
try {
124183
if (LOGGER.isDebugEnabled()) {
@@ -152,21 +211,27 @@ private Path getLibraryExtractDir(final LmdbLibraryConfig lmdbLibraryConfig) {
152211
}
153212

154213
private void cleanUpExtractDir(final Path extractDir) {
155-
try (final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(
156-
extractDir, "lmdbjava-native-library-*.so")) {
214+
cleanUpExtractDir(extractDir, "lmdbjava-native-library-*.so", "LMDB native library");
215+
// The jffi file should not really ever be there because it is deleted once loaded, but just in case
216+
// See com.kenai.jffi.internal.StubLoader
217+
cleanUpExtractDir(extractDir, "jffi*.so", "JFFI native library");
218+
}
157219

158-
for (final Path redundantLibraryFilePath : directoryStream) {
159-
LOGGER.info("Deleting redundant LMDB library file "
160-
+ redundantLibraryFilePath.toAbsolutePath().normalize());
220+
private static void cleanUpExtractDir(final Path extractDir, final String glob, final String name) {
221+
try (final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(extractDir, glob)) {
222+
for (final Path file : directoryStream) {
223+
LOGGER.info("Deleting redundant {} file {}", name, file.toAbsolutePath().normalize());
161224
try {
162-
Files.deleteIfExists(redundantLibraryFilePath);
225+
Files.deleteIfExists(file);
163226
} catch (final IOException e) {
164-
LOGGER.error("Unable to delete file " + extractDir.toAbsolutePath().normalize(), e);
227+
LOGGER.error(() -> LogUtil.message("Unable to delete {} file {}",
228+
name, file.toAbsolutePath().normalize()), e);
165229
// swallow error as these old files don't matter really
166230
}
167231
}
168232
} catch (final IOException e) {
169-
throw new RuntimeException("Error listing contents of " + extractDir.toAbsolutePath().normalize());
233+
throw new RuntimeException(LogUtil.message("Error listing contents of {} with glob '{}'",
234+
extractDir.toAbsolutePath().normalize(), glob));
170235
}
171236
}
172237
}

‎stroom-lmdb/src/main/java/stroom/lmdb/LmdbLibraryConfig.java‎

Lines changed: 40 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,59 +32,89 @@
3232
public class LmdbLibraryConfig extends AbstractConfig implements IsStroomConfig {
3333

3434
static final String SYSTEM_LIBRARY_PATH_PROP_NAME = "providedSystemLibraryPath";
35+
static final String JFFI_LIBRARY_PATH_PROP_NAME = "providedJffiLibraryPath";
3536
static final String EXTRACT_DIR_PROP_NAME = "systemLibraryExtractDir";
3637
static final String DEFAULT_LIBRARY_EXTRACT_SUB_DIR_NAME = "lmdb_library";
3738

3839
// These are dups of org.lmdbjava.Library.LMDB_* but that class is pkg private for some reason.
3940
public static final String LMDB_EXTRACT_DIR_PROP = "lmdbjava.extract.dir";
4041
public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib";
42+
public static final String JFFI_EXTRACT_DIR_PROP = "jffi.extract.dir";
43+
public static final String JFFI_NATIVE_LIB_PROP = "jffi.boot.library.path";
4144

4245
private final String providedSystemLibraryPath;
46+
private final String providedJffiLibraryPath;
4347
private final String systemLibraryExtractDir;
4448

4549
public LmdbLibraryConfig() {
4650
providedSystemLibraryPath = null;
51+
providedJffiLibraryPath = null;
4752
systemLibraryExtractDir = DEFAULT_LIBRARY_EXTRACT_SUB_DIR_NAME;
4853
}
4954

5055
@SuppressWarnings("unused")
5156
@JsonCreator
5257
public LmdbLibraryConfig(@JsonProperty(SYSTEM_LIBRARY_PATH_PROP_NAME) final String providedSystemLibraryPath,
58+
@JsonProperty(JFFI_LIBRARY_PATH_PROP_NAME) final String providedJffiLibraryPath,
5359
@JsonProperty(EXTRACT_DIR_PROP_NAME) final String systemLibraryExtractDir) {
5460
this.providedSystemLibraryPath = providedSystemLibraryPath;
61+
this.providedJffiLibraryPath = providedJffiLibraryPath;
5562
this.systemLibraryExtractDir = systemLibraryExtractDir;
5663
}
5764

5865
@ValidFilePath
5966
@RequiresRestart(RestartScope.SYSTEM)
6067
@JsonProperty(SYSTEM_LIBRARY_PATH_PROP_NAME)
61-
@JsonPropertyDescription("The path to a provided LMDB system library file. If unset the LMDB binary " +
68+
@JsonPropertyDescription(
69+
"The path to a provided LMDB native library file. If unset the LMDB binary " +
6270
"bundled with Stroom will be extracted to 'systemLibraryExtractDir'. This property can be used if " +
6371
"you already have LMDB installed or want to make use of a package manager provided instance. " +
64-
"If you set this property care needs to be taken over version compatibility between the version " +
65-
"of LMDBJava (that Stroom uses to interact with LMDB) and the version of the LMDB binary.")
72+
"If you set this property care needs to be taken over version compatibility between the version " +
73+
"of LMDBJava (that Stroom uses to interact with LMDB) and the version of the LMDB binary. " +
74+
"By default this is unset.")
6675
public String getProvidedSystemLibraryPath() {
6776
return providedSystemLibraryPath;
6877
}
6978

79+
@ValidFilePath
80+
@RequiresRestart(RestartScope.SYSTEM)
81+
@JsonProperty(JFFI_LIBRARY_PATH_PROP_NAME)
82+
@JsonPropertyDescription(
83+
"The path to a provided JFFI native library file, which is used by LMDBJava. If unset the JFFI binary " +
84+
"bundled with Stroom will be extracted to 'systemLibraryExtractDir'. This property can be used if " +
85+
"you provide your own version of JFFI. " +
86+
"If you set this property care needs to be taken over version compatibility between the version " +
87+
"of JFFI and the version of the JFFI binary. " +
88+
"By default this is unset.")
89+
public String getProvidedJffiLibraryPath() {
90+
return providedJffiLibraryPath;
91+
}
92+
7093
@RequiresRestart(RestartScope.SYSTEM)
7194
@JsonProperty(EXTRACT_DIR_PROP_NAME)
72-
@JsonPropertyDescription("The directory to extract the bundled LMDB system library to. Only used if " +
73-
"property providedSystemLibraryPath is not set. On boot Stroom will extract the LMDB binary to this " +
74-
"location. It will also delete old copies of the LMDB system library if found.")
95+
@JsonPropertyDescription(
96+
"The directory to extract the bundled LMDB and JFFI native libraries to. Only used if " +
97+
"property providedSystemLibraryPath or property providedJffiLibraryPath are not set. " +
98+
"On boot Stroom will clear this directory and then extract the native libraries into it. " +
99+
"IMPORTANT: This directory must not have the 'noexec' mount option, or all LMDB use will error. " +
100+
"Also, the contents are ephemeral, so should ideally be mounted using tmpfs or similar.")
75101
public String getSystemLibraryExtractDir() {
76102
return systemLibraryExtractDir;
77103
}
78104

79105
@Override
80106
public String toString() {
81107
return "LmdbLibraryConfig{" +
82-
"providedSystemLibraryPath='" + providedSystemLibraryPath + '\'' +
83-
", systemLibraryExtractDir='" + systemLibraryExtractDir + '\'' +
84-
'}';
108+
"providedSystemLibraryPath='" + providedSystemLibraryPath + '\'' +
109+
", providedJffiLibraryPath='" + providedJffiLibraryPath + '\'' +
110+
", systemLibraryExtractDir='" + systemLibraryExtractDir + '\'' +
111+
'}';
85112
}
86113

87114
public LmdbLibraryConfig withSystemLibraryExtractDir(final String systemLibraryExtractDir) {
88-
return new LmdbLibraryConfig(providedSystemLibraryPath, systemLibraryExtractDir);
115+
return new LmdbLibraryConfig(
116+
providedSystemLibraryPath,
117+
providedJffiLibraryPath,
118+
systemLibraryExtractDir);
89119
}
90120
}

‎unreleased_changes/20260520_163807_374__5549.md‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
* Bug **#5549** : Make jffi extract its native library to the same dir as the LMDB native library file. Add the config prop `providedJffiLibraryPath` to allow for a provided jffi lib.
2+
3+
4+
```sh
5+
# ********************************************************************************
6+
# Issue number: 5549
7+
# Issue title: jnr ffi is extracting a native lib to /tmp which doesn't work if /tmp is mounted with `noexec`
8+
# Issue tags:
9+
# Issue link: https://github.com/gchq/stroom/issues/5549
10+
# ********************************************************************************
11+
12+
# ONLY the top line will be included as a change entry in the CHANGELOG.
13+
# The entry should be in GitHub flavour markdown and should be written on a SINGLE
14+
# line with no hard breaks. You can have multiple change files for a single GitHub issue.
15+
# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
16+
# 'Fixed nasty bug'.
17+
#
18+
# Examples of acceptable entries are:
19+
#
20+
#
21+
# * Bug **#123** : Fix bug with an associated GitHub issue in this repository.
22+
#
23+
# * Bug **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository.
24+
#
25+
# * Feature **#789** : Add new feature X.
26+
#
27+
# * Bug : Fix bug with no associated GitHub issue.
28+
#
29+
#
30+
# Note: The line must start '* XXX ', where 'XXX' is a valid category,
31+
# one of [Bug Feature Refactor Dependency Build].
32+
33+
34+
# --------------------------------------------------------------------------------
35+
# The following is random text to make this file unique for git's change detection
36+
# yycVukXmxnlCAvZEVr03hHqhXGL3WkEckGhJy1PhtNfSCJ2BRoJRC1dsMe0twcmyeGEOCvbPJmRW4kXp
37+
# 6YQZJKkZpswT3rLro2bAzxxjPthiR4uQAPR8Jq5u3nbIC2t3tYFe3foegMPm4vN7x1OYCwDCjU024R5G
38+
# Ig8Agidd79meRXr7mAOPdua5WWAKYaaXS2YrvCuEWkGa1echsybpCtfnRSX029gwPxkjxVa60IiaU4yN
39+
# V9K1MEUbJcjenOkOqiLOb1nNEHl0d7U5EZvGa977HFpR6lsVkX1SZghQRThJBdM8tgktKsPtXGahjqQ4
40+
# pOa4PTzfHVlgWzrWLQ47x6SWNj8ETLjjNwQWC27MTCLzGwHFgxW8pEwXBBWZZuKUw9lDiLLqQ8QQXwdk
41+
# UK5I7gXbUMwZt9py8AKS1K8UuOjNKuR9n78F0PDcnblEk64LazDddDmAGZjmpuRzfY18SecgA8JqwJTq
42+
# 6eTiVEDg3vN3WeTNDEdxTgQfXLEFEJgcj3HEDL9zeTgf88Th0d8UWLGmZFG9PdVW5hEzp2iBPdzvOrlH
43+
# HHem4YK2nIBOTMJgOpMZTvnkSACroCWXNBydjRPaTyDf7VjC2Hne1DF6HADCMjfRy5BiYREQh8U2OfL8
44+
# VHqDXYFm1zGJQYJGpWNmpNxYneEsSywFCz8b5YDQ7Xa1UQBgFS7IfmWNtMY4YzXOQ7LZ1G5dSYBna4wc
45+
# koTY1TYNQyWUfro89rhl5DIfxZ8AaBKV85oQrdrCqrj9jSkRPosDHKQOkHOEoxd7uViBwnd39ViBABR1
46+
# qOVgW9iuNKCPWRP8ShNG4OjKmc46SuPVLgLxlQV08hJzobW4S2iJjFo91D1Ca0eZjj9SN1StDQi1RITD
47+
# yLP63yulM0F3NGYELbKlBKgpglfivNBkltTP7RHFfKYLZRYHYZaF9LK6LBSs1XicHwYw4q0AhlEhmDiL
48+
# JZ57SrHf73IkPK2WWRRuVpECnrqsz55VjHoHbNXir52ERbgSuCSaEhJBYtm8CLjsynSzk4FvzYQxYZk6
49+
# tgAX5UrVB6XmvkbzJj1b2SNhpljklNDzgP7UD6R1Fmu6RvPE8XvCsjKYrUcv4JRPPtZL3lKA6teWdlGU
50+
# TmutuFaEqkrDuRWUNOADPwMz62pCjv2zCI5HfyhaD77oUNMmdfDqpu4AwKRYGKqMw0GYp6a9vVdNjO8l
51+
# q4alUw6IO57BCKfjQ9mweQL6VSfA3adasx3V9AhOehK4pkyWbrOS0FNmmnbDHH5X0J6DZTsk36lkvWbC
52+
# Ns5VU4eG3itbs1txXUPRJVImxNZAYecjKrzWWg5IdixyJyDu723qUVibAelwEfcs1sB06SLR0lwa4tMi
53+
# wAsUpzHkkFCTdXXpAmLPmMQErXDCSitGKqNTV3FW9u1N6TWqhQsmvSY3yG8lroXpUZHeAcFsADdSwNbm
54+
# bt0ic0JEiNlmYnAaMmHUpAunBQmobinpPRIV3eG4qvutSkMQSccjA6kWGA95i1egCFuvIedLbPCvvd5h
55+
# lnuqzdoI2jntj9DcSwu6ySJDLUdzE3XqUl0iQ0yIefU9lV07ql3QmscPQhfisMMjXsIhuo2aMW89Q8wk
56+
# bqHSEjQLjFQAcmgupSulSzY9vI3w1p5N6k8XK1u8pAaHLQtg4FEeNEREoJxigf5Ra5qzfv0IEVC3qsFn
57+
# LPsEqjqmYCNdgipHsPeTrBWMZNobXmiIjeEeKsGECgDp60qbLoz4QlLjRlYSIyad86urBdTpct89G1VG
58+
# 2W8o5uJLzH2snqe7ELaJTPWsgPetTMXC4XXsm1sVuPaQOBwPJNpfznwBRuWicfz7r7Xm3YK2TUJhTLF3
59+
# wFkbXOoseLCqsORnsrZhUO9zDWOFsRu3Xq92AXYhtXs7v8AIVCJZsovHqcKFpCPYEAdBsYNK37nt5H80
60+
# Gzm5p6vhd4NmnVIXPdkvX5dLdc2v07jZmFqfqzgHXqrYKhihDPDVTifJQjo6abXZB0M9ArVEWshR6jyc
61+
# Iskeiz0l99eyuzSlAJ3ilL7WNIK4fi2Rwe2fvU8MqNhPMvxarMxfEQ3NDVrMWHAZWwJab9OkJ1LUaxea
62+
# hyk7D3S35OXuVqupC2dZhdIXSz3yoztzwk9izNCPZStXuM5KjI4PpVsuzrg8KecHO7CUulxGbNHsldRA
63+
# StcTwS8YNeQfliUq11tQkWGxS4WJMZE0mlGoObwYv7mn04s5EgOhKWFEJjIc3oHrLMnXYWI34OaYqxwX
64+
# K70n3u7R8tfWuTeNnhLf8PdEN2k5bVSoSePbPmfxjS5cgnqE9UlXSq2jOVM0ap2ShOAephE9Y59YgP6x
65+
# KxGYUixKTJyW9OxxcZLTHm2MIkfZo376iiGSPjA1TCJNldfHL37kHlYVb9fjQN0OKGPnuDPRkOliIJp0
66+
# --------------------------------------------------------------------------------
67+
68+
```

0 commit comments

Comments
 (0)