Skip to content

Commit 186d684

Browse files
schmallisoschmalliso
committed
DOP-1160: adds deploy stage to world builder, updates list of projects
1 parent d42f8c9 commit 186d684

File tree

3 files changed

+150
-82
lines changed

3 files changed

+150
-82
lines changed

‎tools/world-builder/README.rst

+10-9
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,19 @@
1-
===================================
2-
MongoDB Documentation World Builder
3-
===================================
1+
========================================================
2+
MongoDB Documentation World Builder for Legacy Toolchain
3+
========================================================
44

5-
This directory contains the MongoDB Documentation World Builder. The World
6-
Builder provides a centralized way of building all MongoDB documentation
7-
projects.
5+
This directory contains the MongoDB Documentation World Builder. The
6+
World Builder provides a centralized way of building **and deploying**
7+
all MongoDB documentation projects that build with the legacy toolchain
8+
(i.e. giza/sphinx).
89

910
Quick Start
1011
-----------
1112

12-
Run ``./build.py`` to build all MongoDB documentation projects.
13+
Run ``./build.py`` to build and deploy all MongoDB documentation projects.
1314

14-
Building Specific Targets
15-
-------------------------
15+
Build and Deploy Specific Targets
16+
---------------------------------
1617

1718
The World Builder can also fetch and build specific projects using
1819
shell-style glob expressions on git repository names.

‎tools/world-builder/build.py

+58-31
Original file line numberDiff line numberDiff line change
@@ -7,67 +7,81 @@
77
import yaml
88
from typing import Any, Dict, Optional, Set, List
99

10-
PAT_URL = re.compile('.+/(.+).git')
10+
PAT_URL = re.compile(".+/(.+).git")
1111

1212

1313
class BuildTask:
14-
def __init__(self, name: str, branches: List[str], build_dir: str, build_command: str) -> None:
14+
def __init__(
15+
self,
16+
name: str,
17+
branches: List[str],
18+
build_dir: str,
19+
build_command: str,
20+
deploy_command: str,
21+
) -> None:
1522
self.name = name
1623
self.branches = branches
1724
self.build_dir = build_dir
1825
self.build_command = build_command
19-
self.built_flag_file = os.path.join(self.build_dir, '.built')
26+
self.deploy_command = deploy_command
27+
self.built_flag_file = os.path.join(self.build_dir, ".built")
2028

2129
def build(self) -> None:
22-
subprocess.check_call(['git', 'clean', '-xfd'], cwd=self.build_dir)
30+
subprocess.check_call(["git", "clean", "-xfd"], cwd=self.build_dir)
2331
# Some of our properties die if these directories do not exist
24-
for path in ('build', 'build/master', 'build/public', 'build/public/master'):
32+
for path in ("build", "build/master", "build/public", "build/public/master"):
2533
try:
2634
os.mkdir(os.path.join(self.build_dir, path))
2735
except OSError:
2836
pass
2937

3038
for branch in self.branches:
31-
print('Building ' + self.build_dir)
32-
subprocess.check_call(['git', 'checkout', '-q', branch], cwd=self.build_dir)
33-
subprocess.check_call(['git', 'pull', '--ff-only', '-q'], cwd=self.build_dir)
39+
print("Building " + self.build_dir)
40+
subprocess.check_call(["git", "checkout", "-q", branch], cwd=self.build_dir)
41+
subprocess.check_call(
42+
["git", "pull", "--ff-only", "-q"], cwd=self.build_dir
43+
)
3444
subprocess.check_call(self.build_command, shell=True, cwd=self.build_dir)
3545

46+
def deploy(self) -> None:
47+
for branch in self.branches:
48+
print("Deploying " + self.build_dir + branch)
49+
subprocess.check_call(["git", "checkout", "-q", branch], cwd=self.build_dir)
50+
subprocess.check_call(self.deploy_command, shell=True, cwd=self.build_dir)
51+
3652

3753
def load_branches_from_yaml(path: str) -> List[str]:
38-
with open(path, 'r') as f:
54+
with open(path, "r") as f:
3955
loaded = yaml.safe_load(f)
4056

41-
return loaded['git']['branches']['published']
57+
return loaded["git"]["branches"]["published"]
4258

4359

4460
def initialize_project(project: Dict[str, Any]) -> BuildTask:
45-
git_url = project['git']
46-
project_name = project.get('name', None)
61+
git_url = project["git"]
62+
project_name = project.get("name", None)
4763
if not project_name:
4864
match = PAT_URL.match(git_url)
4965
if not match:
50-
raise ValueError('Cannot determine project name: {}'.format(git_url))
66+
raise ValueError("Cannot determine project name: {}".format(git_url))
5167
project_name = match.group(1)
5268

53-
branches = project['branches']
69+
branches = project["branches"]
5470
if isinstance(branches, str):
5571
branches = load_branches_from_yaml(branches)
5672

57-
output_dir = os.path.join('build', project_name)
73+
output_dir = os.path.join("build", project_name)
5874
if not os.path.isdir(output_dir):
59-
print('Cloning {}'.format(git_url))
60-
subprocess.check_call(['git', 'clone', '-q', git_url, output_dir])
75+
print("Cloning {}".format(git_url))
76+
subprocess.check_call(["git", "clone", "-q", git_url, output_dir])
6177

6278
return BuildTask(
63-
project_name,
64-
branches,
65-
output_dir,
66-
project['build'])
79+
project_name, branches, output_dir, project["build"], project["deploy"]
80+
)
6781

6882

6983
def main(path: str, projects: Optional[Set[str]]) -> None:
70-
with open(path, 'r') as f:
84+
with open(path, "r") as f:
7185
data = json.load(f)
7286

7387
tasks = []
@@ -76,24 +90,37 @@ def main(path: str, projects: Optional[Set[str]]) -> None:
7690
if projects is None or task.name in projects:
7791
tasks.append(task)
7892

79-
errors = [] # type: List[str]
93+
build_errors = [] # type: List[str]
8094
for task in tasks:
8195
try:
8296
task.build()
8397
except subprocess.SubprocessError:
84-
errors.append(task.build_dir)
85-
print('Failed to build in {}'.format(task.build_dir))
98+
build_errors.append(task.build_dir)
99+
print("Failed to build in {}".format(task.build_dir))
100+
tasks.remove(task) # so that we don't deploy the unbuilt thing
101+
102+
if build_errors:
103+
print("\nFailed to build:")
104+
for d in build_errors:
105+
print(" " + d)
106+
107+
deploy_errors = [] # type: List[str]
108+
for task in tasks:
109+
try:
110+
task.deploy()
111+
except subprocess.SubprocessError:
112+
deploy_errors.append(task.build)
86113

87-
if errors:
88-
print('\nFailed to build:')
89-
for d in errors:
90-
print(' ' + d)
114+
if deploy_errors:
115+
print("\nFailed to deploy:")
116+
for d in deploy_errors:
117+
print(" " + str(d))
91118

92119

93-
if __name__ == '__main__':
120+
if __name__ == "__main__":
94121
if len(sys.argv) > 1:
95122
projects = set(sys.argv[1:]) # type: Optional[Set[str]]
96123
else:
97124
projects = None
98125

99-
main('projects.json', projects)
126+
main("projects.json", projects)

‎tools/world-builder/projects.json

+82-42
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,120 @@
11
[
22
{
3-
"git": "https://github.com/mongodb/docs-bi-connector.git",
4-
"branches": "../../data/bi-connector-published-branches.yaml",
5-
"build": "mkdir -p build/master && make publish"
3+
"git": "[email protected]:10gen/cloud-docs.git",
4+
"branches": ["master"],
5+
"build": "make publish",
6+
"deploy": "make deploy"
67
},
78
{
8-
"git": "https://github.com/mongodb/docs-spark-connector.git",
9-
"branches": "../../data/spark-connector-published-branches.yaml",
10-
"build": "giza make publish"
9+
"name": "atlasOSB",
10+
"git": "[email protected]:10gen/cloud-docs-osb.git",
11+
"branches": "../../data/atlas-open-service-broker-published-branches.yaml",
12+
"build": "make publish",
13+
"deploy": "make deploy"
1114
},
1215
{
13-
"git": "https://github.com/mongodb/docs.git",
14-
"branches": "../../data/manual-published-branches.yaml",
15-
"build": "make publish"
16+
"git": "https://github.com/mongodb/docs-bi-connector.git",
17+
"branches": "../../data/bi-connector-published-branches.yaml",
18+
"build": "make publish",
19+
"deploy": "make deploy"
1620
},
1721
{
18-
"name": "mms-onprem",
19-
"git": "[email protected]:10gen/mms-docs.git",
20-
"branches": "../../data/mms-published-branches.yaml",
21-
"build": "giza make publish-onprem"
22+
"git": "[email protected]:10gen/docs-charts.git",
23+
"branches": "../../data/charts-published-branches.yaml",
24+
"build": "make publish",
25+
"deploy": "make deploy"
2226
},
2327
{
2428
"name": "mms-cloud",
2529
"git": "[email protected]:10gen/mms-docs.git",
2630
"branches": ["master"],
27-
"build": "make publish-cloud"
28-
},
29-
{
30-
"git": "[email protected]:10gen/baas-docs.git",
31-
"branches": "../../data/stitch-published-branches.yaml",
32-
"build": "make publish"
31+
"build": "make publish-cloud",
32+
"deploy": "make deploy-cloud"
3333
},
3434
{
35-
"git": "https://github.com/mongodb/docs-compass.git",
35+
"git": "git@github.com:mongodb/docs-compass.git",
3636
"branches": "../../data/compass-published-branches.yaml",
37-
"build": "make publish"
37+
"build": "make publish",
38+
"deploy": "make deploy"
3839
},
3940
{
40-
"git": "[email protected]:10gen/docs-charts.git",
41-
"branches": "../../data/charts-published-branches.yaml",
42-
"build": "make publish"
41+
"git": "[email protected]:mongodb/docs-commandline-tools.git",
42+
"branches": ["master"],
43+
"build": "make publish",
44+
"deploy": "make deploy"
4345
},
4446
{
45-
"git": "[email protected]:10gen/cloud-docs.git",
47+
"name": "guides",
48+
"git": "[email protected]:10gen/docs-tutorials.git",
4649
"branches": ["master"],
47-
"build": "make publish"
50+
"build": "make publish",
51+
"deploy": "make deploy"
4852
},
4953
{
50-
"git": "https://github.com/mongodb/docs-php-library.git",
51-
"branches": "../../data/php-library-published-branches.yaml",
52-
"build": "make publish"
54+
"git": "[email protected]:10gen/docs-k8s-operator.git",
55+
"branches": "../../data/kubernetes-operator-published-branches.yaml",
56+
"build": "make publish",
57+
"deploy": "make deploy"
5358
},
5459
{
55-
"git": "https://github.com/mongodb/docs-ruby.git",
56-
"branches": "../../data/ruby-driver-published-branches.yaml",
57-
"build": "make publish"
60+
"git": "[email protected]:10gen/docs-kafka-connector.git",
61+
"branches": "../../data/kafka-connector-published-branches.yaml",
62+
"build": "make publish",
63+
"deploy": "make deploy"
5864
},
5965
{
60-
"git": "[email protected]:10gen/docs-mongoid.git",
61-
"branches": "../../data/mongoid-published-branches.yaml",
62-
"build": "make publish"
66+
"git": "[email protected]:mongodb/docs-landing.git",
67+
"branches": ["master"],
68+
"build": "make build",
69+
"deploy": "make deploy"
6370
},
6471
{
6572
"git": "[email protected]:mongodb/docs-meta.git",
6673
"branches": ["master"],
67-
"build": "make publish"
74+
"build": "make publish",
75+
"deploy": "make deploy"
6876
},
6977
{
70-
"git": "[email protected]:mongodb/docs-ecosystem.git",
71-
"branches": ["master"],
72-
"build": "make publish"
78+
"git": "[email protected]:10gen/docs-mongocli.git",
79+
"branches": "../../data/mongocli-published-branches.yaml",
80+
"build": "make publish",
81+
"deploy": "make deploy"
82+
},
83+
{
84+
"name": "mms-onprem",
85+
"git": "[email protected]:10gen/mms-docs.git",
86+
"branches": "../../data/mms-published-branches.yaml",
87+
"build": "make publish-onprem",
88+
"deploy": "make deploy-opsmgr"
89+
},
90+
{
91+
"git": "https://github.com/mongodb/docs-spark-connector.git",
92+
"branches": "../../data/spark-connector-published-branches.yaml",
93+
"build": "make publish",
94+
"deploy": "make deploy"
95+
},
96+
{
97+
"git": "[email protected]:mongodb/docs-php-library.git",
98+
"branches": "../../data/php-library-published-branches.yaml",
99+
"build": "true",
100+
"deploy": "make deploy"
101+
},
102+
{
103+
"git": "[email protected]:mongodb/docs-ruby.git",
104+
"branches": "../../data/ruby-driver-published-branches",
105+
"build": "make publish",
106+
"deploy": "make deploy"
107+
},
108+
{
109+
"git": "[email protected]:mongodb/docs.git",
110+
"branches": "../../data/manual-published-branches",
111+
"build": "make publish",
112+
"deploy": "make deploy"
73113
},
74114
{
75-
"git": "https://github.com/mongodb/docs-landing.git",
115+
"git": "git@github.com:mongodb/docs-mongodb-shell.git",
76116
"branches": ["master"],
77-
"build": "",
78-
"docs-tools": false
117+
"build": "make publish",
118+
"deploy": "make deploy"
79119
}
80120
]

0 commit comments

Comments
 (0)