Skip to content

Commit 73149d4

Browse files
ccgsnetccgsnet
authored
Merge pull request #1058 from singnet/json-config
[Config] Add `--config=FILE` argument
2 parents 50e24ff + 0799311 commit 73149d4

28 files changed

Lines changed: 827 additions & 176 deletions

‎config/example.json‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
{
2+
"schema_version": "1.0",
3+
"atomdb": {
4+
"type": "redismongodb",
5+
"redis": {
6+
"image": "redis:7.2.3-alpine",
7+
"endpoint": "localhost:40020",
8+
"cluster": false,
9+
"nodes": [
10+
{
11+
"context": "default",
12+
"ip": "localhost",
13+
"username": "arturgontijo"
14+
}
15+
]
16+
},
17+
"mongodb": {
18+
"image": "mongodb/mongodb-community-server:8.2-ubuntu2204",
19+
"endpoint": "localhost:40021",
20+
"username": "admin",
21+
"password": "admin",
22+
"cluster": false,
23+
"cluster_secret_key": "8UDJSgpUCaVOTQG",
24+
"nodes": [
25+
{
26+
"context": "default",
27+
"ip": "localhost",
28+
"username": "arturgontijo"
29+
}
30+
]
31+
},
32+
"morkdb": {
33+
"image": "trueagi/das:mork-server-1.0.4",
34+
"endpoint": "localhost:40022"
35+
},
36+
"remote_peers": [
37+
{
38+
"uid": "peer1",
39+
"type": "redismongodb",
40+
"context": "remotedb_test_peer1_",
41+
"mongodb": {
42+
"endpoint": "localhost:40021",
43+
"username": "admin",
44+
"password": "admin"
45+
},
46+
"redis": {
47+
"endpoint": "localhost:40020",
48+
"cluster": false
49+
},
50+
"local_persistence": {
51+
"type": "morkdb",
52+
"context": "remotedb_test_peer1_local_",
53+
"mongodb": {
54+
"endpoint": "localhost:40021",
55+
"username": "admin",
56+
"password": "admin"
57+
},
58+
"morkdb": {
59+
"endpoint": "localhost:40022"
60+
}
61+
}
62+
},
63+
{
64+
"uid": "peer2",
65+
"type": "inmemorydb",
66+
"context": "remotedb_test_peer2_",
67+
"local_persistence": {
68+
"type": "inmemorydb",
69+
"context": "remotedb_test_peer2_local_"
70+
}
71+
}
72+
]
73+
},
74+
"loaders": {
75+
"metta": {
76+
"image": "trueagi/das:1.0.0-metta-parser"
77+
},
78+
"morkdb": {
79+
"image": "trueagi/das:mork-loader-1.0.4"
80+
}
81+
},
82+
"agents": {
83+
"query": {
84+
"endpoint": "localhost:40002",
85+
"ports_range": "42000:42999"
86+
},
87+
"link_creation": {
88+
"endpoint": "localhost:40003",
89+
"ports_range": "43000:43999"
90+
},
91+
"inference": {
92+
"endpoint": "localhost:40004",
93+
"ports_range": "44000:44999"
94+
},
95+
"evolution": {
96+
"endpoint": "localhost:40005",
97+
"ports_range": "45000:45999"
98+
}
99+
},
100+
"brokers": {
101+
"attention": {
102+
"endpoint": "localhost:40001"
103+
},
104+
"context": {
105+
"endpoint": "localhost:40006",
106+
"ports_range": "46000:46999"
107+
},
108+
"atomdb": {
109+
"endpoint": "localhost:40007",
110+
"ports_range": "47000:47999"
111+
}
112+
},
113+
"params": {
114+
"query": {
115+
"max_answers": 100,
116+
"max_bundle_size": 1000,
117+
"count_flag": false,
118+
"attention_update_flag": false,
119+
"unique_assignment_flag": true,
120+
"positive_importance_flag": false,
121+
"populate_metta_mapping": true,
122+
"use_metta_as_query_tokens": true
123+
},
124+
"link_creation": {
125+
"repeat_count": 1,
126+
"query_interval": 0,
127+
"query_timeout": 0
128+
},
129+
"evolution": {
130+
"elitism_rate": 0.08,
131+
"max_generations": 10,
132+
"population_size": 50,
133+
"selection_rate": 0.1,
134+
"total_attention_tokens": 100000
135+
},
136+
"context": {
137+
"context": "context",
138+
"use_cache": true,
139+
"enforce_cache_recreation": false,
140+
"initial_rent_rate": 0.25,
141+
"initial_spreading_rate_lowerbound": 0.5,
142+
"initial_spreading_rate_upperbound": 0.7
143+
}
144+
}
145+
}

‎src/MODULE.bazel‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,3 +103,12 @@ register_toolchains("@rules_pkg_rpmbuild//:all")
103103

104104
# Dependency for JSON
105105
bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1")
106+
107+
# Work around BCR rules_kotlin patch causing "Permission denied" on MODULE.bazel when
108+
# extracting in Docker (extracted files can be read-only; see bazelbuild/bazel#7369).
109+
# Override with the same archive but no patches so the patch step is skipped.
110+
archive_override(
111+
module_name = "rules_kotlin",
112+
integrity = "sha256-O3cpdv7Hvc2h2EudObF2WJQkwEfrIXW+0JqsYw5Qr0M=",
113+
urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.6/rules_kotlin-v1.9.6.tar.gz"],
114+
)

‎src/MODULE.bazel.lock‎

Lines changed: 1 addition & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/atomdb/AtomDBAPITypes.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class AtomDocument {
6767
virtual bool contains(const string& key) = 0;
6868
};
6969

70-
enum ATOMDB_TYPE { REDIS_MONGODB, MORKDB };
70+
enum ATOMDB_TYPE { REDIS_MONGODB, MORKDB, REMOTEDB };
7171

7272
} // namespace atomdb_api_types
7373
} // namespace atomdb

‎src/atomdb/morkdb/MorkDB.cc‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -97,25 +97,23 @@ string MorkClient::url_encode(const string& value) {
9797
// <--
9898

9999
// --> MorkDB : RedisMongoDB(context, skip_redis = true)
100-
MorkDB::MorkDB(const string& context) : RedisMongoDB(context, true) {
100+
MorkDB::MorkDB(const string& context, const JsonConfig& config) : RedisMongoDB(context, true, config) {
101101
bool disable_cache = true;
102102
this->atomdb_cache = disable_cache ? nullptr : AtomDBCacheSingleton::get_instance();
103-
mork_setup();
103+
mork_setup(config);
104104
}
105105

106106
MorkDB::~MorkDB() {}
107107

108108
bool MorkDB::allow_nested_indexing() { return true; }
109109

110-
void MorkDB::mork_setup() {
110+
void MorkDB::mork_setup(const JsonConfig& config) {
111111
string host = Utils::get_environment("DAS_MORK_HOSTNAME");
112112
string port = Utils::get_environment("DAS_MORK_PORT");
113-
string address = host + ":" + port;
113+
string address = config.at_path("morkdb.endpoint").get_or<string>(host + ":" + port);
114114

115-
if (host == "" || port == "") {
116-
Utils::error(
117-
"You need to set MORK access info as environment variables: DAS_MORK_HOSTNAME, "
118-
"DAS_MORK_PORT");
115+
if (address == "" || address == ":") {
116+
Utils::error("You need to set MORK access info as configuration: morkdb.endpoint");
119117
}
120118

121119
try {

‎src/atomdb/morkdb/MorkDB.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@
1515

1616
#include "AtomDB.h"
1717
#include "AtomDBCacheSingleton.h"
18+
#include "JsonConfig.h"
1819
#include "MorkDBAPITypes.h"
1920
#include "RedisMongoDB.h"
2021

22+
using namespace commons;
23+
2124
namespace atomdb {
2225

2326
#define MORKDB_CHUNK_SIZE 5000
@@ -39,7 +42,7 @@ class MorkClient {
3942

4043
class MorkDB : public RedisMongoDB {
4144
public:
42-
MorkDB(const string& context = "");
45+
MorkDB(const string& context = "", const JsonConfig& config = JsonConfig());
4346
~MorkDB();
4447

4548
bool allow_nested_indexing() override;
@@ -62,7 +65,7 @@ class MorkDB : public RedisMongoDB {
6265
shared_ptr<AtomDBCache> atomdb_cache;
6366
shared_ptr<MorkClient> mork_client;
6467

65-
void mork_setup();
68+
void mork_setup(const JsonConfig& config);
6669
};
6770

6871
} // namespace atomdb

‎src/atomdb/redis_mongodb/RedisMongoDB.cc‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@ string RedisMongoDB::MONGODB_FIELD_NAME[MONGODB_FIELD::size];
3232
uint RedisMongoDB::MONGODB_CHUNK_SIZE;
3333
mongocxx::instance RedisMongoDB::MONGODB_INSTANCE;
3434

35-
RedisMongoDB::RedisMongoDB(const string& context, bool skip_redis) {
35+
RedisMongoDB::RedisMongoDB(const string& context, bool skip_redis, const JsonConfig& config) {
3636
initialize_statics(context, skip_redis);
37-
mongodb_setup();
37+
mongodb_setup(config);
3838
load_pattern_index_schema();
3939
bool disable_cache = true;
4040
this->atomdb_cache = disable_cache ? nullptr : AtomDBCacheSingleton::get_instance();
41-
redis_setup();
41+
redis_setup(config);
4242
this->patterns_next_score.store(get_next_score(REDIS_PATTERNS_PREFIX + ":next_score"));
4343
this->incoming_set_next_score.store(get_next_score(REDIS_INCOMING_PREFIX + ":next_score"));
4444
}
@@ -50,17 +50,28 @@ RedisMongoDB::~RedisMongoDB() {
5050

5151
bool RedisMongoDB::allow_nested_indexing() { return false; }
5252

53-
void RedisMongoDB::redis_setup() {
53+
void RedisMongoDB::redis_setup(const JsonConfig& config) {
5454
if (SKIP_REDIS) return;
5555

5656
string host = Utils::get_environment("DAS_REDIS_HOSTNAME");
5757
string port = Utils::get_environment("DAS_REDIS_PORT");
58-
string address = host + ":" + port;
59-
string cluster = Utils::get_environment("DAS_USE_REDIS_CLUSTER");
58+
59+
string address = config.at_path("redis.endpoint").get_or<string>("");
60+
61+
if (!address.empty()) {
62+
auto tokens = Utils::split(address, ':');
63+
host = tokens[0];
64+
port = tokens[1];
65+
} else {
66+
address = host + ":" + port;
67+
}
68+
69+
string cluster =
70+
config.at_path("redis.cluster").get_or<string>(Utils::get_environment("DAS_USE_REDIS_CLUSTER"));
6071
std::transform(cluster.begin(), cluster.end(), cluster.begin(), ::toupper);
6172
this->cluster_flag = (cluster == "TRUE");
6273

63-
if (host == "" || port == "") {
74+
if (address == "" || address == ":") {
6475
Utils::error(
6576
"You need to set Redis access info as environment variables: DAS_REDIS_HOSTNAME, "
6677
"DAS_REDIS_PORT and DAS_USE_REDIS_CLUSTER");
@@ -70,25 +81,29 @@ void RedisMongoDB::redis_setup() {
7081
<< address);
7182
}
7283

73-
void RedisMongoDB::mongodb_setup() {
84+
void RedisMongoDB::mongodb_setup(const JsonConfig& config) {
7485
string host = Utils::get_environment("DAS_MONGODB_HOSTNAME");
7586
string port = Utils::get_environment("DAS_MONGODB_PORT");
76-
string user = Utils::get_environment("DAS_MONGODB_USERNAME");
77-
string password = Utils::get_environment("DAS_MONGODB_PASSWORD");
87+
string address = config.at_path("mongodb.endpoint").get_or<string>(host + ":" + port);
88+
string user = config.at_path("mongodb.username")
89+
.get_or<string>(Utils::get_environment("DAS_MONGODB_USERNAME"));
90+
string password = config.at_path("mongodb.password")
91+
.get_or<string>(Utils::get_environment("DAS_MONGODB_PASSWORD"));
7892
try {
79-
uint chunk_size = Utils::string_to_int(Utils::get_environment("DAS_MONGODB_CHUNK_SIZE"));
93+
uint chunk_size =
94+
config.at_path("mongodb.chunk_size")
95+
.get_or<uint>(Utils::string_to_int(Utils::get_environment("DAS_MONGODB_CHUNK_SIZE")));
8096
if (chunk_size > 0) {
8197
MONGODB_CHUNK_SIZE = chunk_size;
8298
}
8399
} catch (const std::exception& e) {
84100
LOG_INFO("Using default MongoDB chunk size: " + to_string(MONGODB_CHUNK_SIZE));
85101
}
86-
if (host == "" || port == "" || user == "" || password == "") {
102+
if (address == "" || address == ":" || user == "" || password == "") {
87103
Utils::error(
88104
string("You need to set MongoDB access info as environment variables: ") +
89105
"DAS_MONGODB_HOSTNAME, DAS_MONGODB_PORT, DAS_MONGODB_USERNAME and DAS_MONGODB_PASSWORD");
90106
}
91-
string address = host + ":" + port;
92107
string url = "mongodb://" + user + ":" + password + "@" + address;
93108

94109
try {

‎src/atomdb/redis_mongodb/RedisMongoDB.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,23 @@
1616

1717
#include "AtomDB.h"
1818
#include "AtomDBCacheSingleton.h"
19+
#include "JsonConfig.h"
1920
#include "RedisContext.h"
2021
#include "RedisContextPool.h"
2122
#include "RedisMongoDBAPITypes.h"
2223

2324
using namespace std;
25+
using namespace commons;
2426

2527
namespace atomdb {
2628

2729
enum MONGODB_FIELD { ID = 0, NAME, TARGETS, NAMED_TYPE, size };
2830

2931
class RedisMongoDB : public AtomDB {
3032
public:
31-
RedisMongoDB(const string& context = "", bool skip_redis = false);
33+
RedisMongoDB(const string& context = "",
34+
bool skip_redis = false,
35+
const JsonConfig& config = JsonConfig());
3236
~RedisMongoDB();
3337

3438
bool allow_nested_indexing() override;
@@ -192,8 +196,8 @@ class RedisMongoDB : public AtomDB {
192196
vector<string> match_pattern_index_schema(const Link* link);
193197
vector<vector<string>> index_entries_combinations(unsigned int arity);
194198

195-
void redis_setup();
196-
void mongodb_setup();
199+
void redis_setup(const JsonConfig& config);
200+
void mongodb_setup(const JsonConfig& config);
197201
};
198202

199203
} // namespace atomdb

0 commit comments

Comments
 (0)