Skip to content

Commit a69c6e5

Browse files
barakberbarakber
committed
more work
1 parent 7822e42 commit a69c6e5

File tree

24 files changed

+1313
-183
lines changed

24 files changed

+1313
-183
lines changed

‎README.md

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# deployment
2+
Atidot Deployment to AWS (CloudFormation)
3+
# Motivation
4+
5+
- Cloudformation templates are complex error-prone JSONs
6+
- Use a simpler format (`Deployment`) and translate to `Cloudformation`
7+
- Decompose AWS deployment to modular subcomponents
8+
- Compiler checks for `Cloudformation` correctness (even before `awscli` or `EC2 Console`)
9+
- CLI / Editor
10+
~~~
11+
# Build
12+
CLI (Command Line Interface)
13+
~~~shell
14+
barak@berkos:~/deployment/build (master) $ make
15+
nix-build --cores 0 -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/19.09.tar.gz nix/default.nix
16+
/nix/store/40fn6kv14yhdf6dpabk4mlh3nz83iw07-deployment
17+
18+
barak@berkos:~/deployment/build (master) $ ./result/bin/atidot-deployment
19+
Missing: COMMAND
20+
21+
Usage: atidot-deployment COMMAND
22+
Atidot Deployment
23+
~~~
24+
Editor
25+
~~~shell
26+
barak@berkos:~/deployment/build (master) $ make editor
27+
nix-build --cores 0 -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/19.09.tar.gz -A ghcjs.atidot-deployment-editor nix/editor.nix
28+
/nix/store/l8m8jwkhqd0lc11z8cnwc3avq50n9l8p-atidot-deployment-editor-0.1.0.0-js-unknown-ghcjs
29+
barak@berkos:~/deployment/build (master) $ chromium ./result/bin/atidot-deployment-editor.jsexe/index.html
30+
~~~
31+
# Deploy using `awscli` V2
32+
~~~shell
33+
awscli-chrootenv:barak@berkos:~/deployment/build$ ./result/bin/atidot-deployment example | ./result/bin/atidot-deployment translate > cf.json
34+
awscli-chrootenv:barak@berkos:~/deployment/build$ aws cloudformation create-stack --stack-name atidot-test --template-body file://cf.json --capabilities CAPABILITY_IAM
35+
{
36+
"StackId": "arn:aws:cloudformation:us-east-1:683395580497:stack/atidot-test/80481d70-6210-11ea-aec6-0e46d57ace87"
37+
}
38+
~~~
39+
40+
# Command Line Interface
41+
Output an example Atidot `Deployment` JSON
42+
~~~shell
43+
barak@berkos:~/deployment/build (master) $ ./result/bin/atidot-deployment example | jq -c
44+
{"cluster":{"Cluster":{"tasks":[{"use":true,"Container":{"disk":4000,"mem":8000,"name":"undins","cpu":2}},{"use":true,"Container":{"disk":4000,"mem":8000,"name":"jobrunner","cpu":2}},{"use":true,"Container":{"disk":4000,"mem":8000,"name":"api","cpu":2}},{"use":true,"Container":{"disk":4000,"mem":8000,"name":"log-server","cpu":2}},{"use":true,"Container":{"disk":4000,"mem":8000,"name":"rabbitmq","cpu":2}}],"name":"Cluster"},"use":true},"cd":{"use":true,"CD":{"name":"CD"}},"registry":{"use":true,"Registry":{"name":"Registry"}},"vpc":{"use":true,"VPC":{"name":"VPC"}},"region":"us-east-2"}
45+
15:09 barak@berkos:~/Development/atidot/deployment/build (master) $
46+
~~~
47+
Translate an Atidot `Deployment` to Cloudformation Template
48+
~~~shell
49+
barak@berkos:~/deployment/build (master) $ ./result/bin/atidot-deployment example | ./result/bin/atidot-deployment translate | jq -c
50+
{"AWSTemplateFormatVersion":"2010-09-09","Description":"Atidot ECS Deployment","Resources":{"LogServerTask":{"DependsOn":["VPC"],"Type":"AWS::ECS::TaskDefinition","Properties":{"ContainerDefinitions":[{"Image":{"Fn::Join":["",[{"Ref":"AWS::AccountId"},".dkr.ecr.us-east-2.amazonaws.com/",{"Ref":"Registry"},":","latest"]]},"Memory":8000,"Name":"log-server","Cpu":2}]}},"LogServerService":{"DependsOn":["VPC"],"Type":"AWS::ECS::Service","Properties":{"Cluster":{"Ref":"Cluster"},"DesiredCount":1,"ServiceName":"log-server","LaunchType":"EC2","TaskDefinition":{"Ref":"LogServerTask"}}},"RabbitmqTask":{"DependsOn":["VPC"],"Type":"AWS::ECS::TaskDefinition","Properties":{"ContainerDefinitions":[{"Image":{"Fn::Join":["",[{"Ref":"AWS::AccountId"},".dkr.ecr.us-east-2.amazonaws.com/",{"Ref":"Registry"},":","latest"]]},"Memory":8000,"Name":"rabbitmq","Cpu":2}]}},"Cluster":{"DependsOn":["VPC"],"Type":"AWS::ECS::Cluster","Properties":{}},"JenkinsRole":{"Type":"AWS::IAM::Role","Properties":{"AssumeRolePolicyDocument":{"Statement":[{"Effect":"Allow","Action":"sts:AssumeRole","Principal":{"Service":["ec2.amazonaws.com"]},"Sid":""}]},"Path":"/"}},"ApiTask":{"DependsOn":["VPC"],"Type":"AWS::ECS::TaskDefinition","Properties":{"ContainerDefinitions":[{"Image":{"Fn::Join":["",[{"Ref":"AWS::AccountId"},".dkr.ecr.us-east-2.amazonaws.com/",{"Ref":"Registry"},":","latest"]]},"Memory":8000,"Name":"api","Cpu":2}]}},"Profile":{"Type":"AWS::IAM::InstanceProfile","Properties":{"Roles":[{"Ref":"Ec2Role"}],"Path":"/"}},"RabbitmqService":{"DependsOn":["VPC"],"Type":"AWS::ECS::Service","Properties":{"Cluster":{"Ref":"Cluster"},"DesiredCount":1,"ServiceName":"rabbitmq","LaunchType":"EC2","TaskDefinition":{"Ref":"RabbitmqTask"}}},"CD":{"Type":"AWS::ECS::Cluster","Properties":{}},"JobrunnerService":{"DependsOn":["VPC"],"Type":"AWS::ECS::Service","Properties":{"Cluster":{"Ref":"Cluster"},"DesiredCount":1,"ServiceName":"jobrunner","LaunchType":"EC2","TaskDefinition":{"Ref":"JobrunnerTask"}}},"JobrunnerTask":{"DependsOn":["VPC"],"Type":"AWS::ECS::TaskDefinition","Properties":{"ContainerDefinitions":[{"Image":{"Fn::Join":["",[{"Ref":"AWS::AccountId"},".dkr.ecr.us-east-2.amazonaws.com/",{"Ref":"Registry"},":","latest"]]},"Memory":8000,"Name":"jobrunner","Cpu":2}]}},"VPC":{"Type":"AWS::EC2::VPC","Properties":{"CidrBlock":"10.0.0.0/24"}},"Registry":{"Type":"AWS::ECR::Repository","Properties":{"RepositoryPolicyText":{"Statement":[{"Effect":"Allow","Action":["ecr:GetDownloadUrlForLayer","ecr:BatchGetImage","ecr:BatchCheckLayerAvailability","ecr:PutImage","ecr:InitiateLayerUpload","ecr:UploadLayerPart","ecr:CompleteLayerUpload"],"Principal":{"AWS":[{"Fn::Join":["",["arn:aws:iam::",{"Ref":"AWS::AccountId"},":user/","atidot"]]}]},"Sid":"AllowPushPull"}],"Version":"2008-10-17"},"RepositoryName":"registry"}},"ApiService":{"DependsOn":["VPC"],"Type":"AWS::ECS::Service","Properties":{"Cluster":{"Ref":"Cluster"},"DesiredCount":1,"ServiceName":"api","LaunchType":"EC2","TaskDefinition":{"Ref":"ApiTask"}}},"Ec2Role":{"Type":"AWS::IAM::Role","Properties":{"AssumeRolePolicyDocument":{"Statement":[{"Effect":"Allow","Action":["sts:AssumeRole"],"Principal":{"Service":["ec2.amazonaws.com"]}}]},"Path":"/","Policies":[{"PolicyDocument":{"Statement":[{"Effect":"Allow","Action":["ecs:CreateCluster","ecs:RegisterContainerInstance","ecs:DeregisterConta
51+
~~~
52+
53+
# Atidot `Deployment`
54+
Example
55+
56+
~~~shell
57+
15:14 barak@berkos:~/Development/atidot/deployment/build (master) $ ./result/bin/atidot-deployment example
58+
{
59+
"cluster": {
60+
"Cluster": {
61+
"tasks": [
62+
{
63+
"use": true,
64+
"Container": {
65+
"disk": 4000,
66+
"mem": 8000,
67+
"name": "undins",
68+
"cpu": 2
69+
}
70+
},
71+
{
72+
"use": true,
73+
"Container": {
74+
"disk": 4000,
75+
"mem": 8000,
76+
"name": "jobrunner",
77+
"cpu": 2
78+
}
79+
},
80+
{
81+
"use": true,
82+
"Container": {
83+
"disk": 4000,
84+
"mem": 8000,
85+
"name": "api",
86+
"cpu": 2
87+
}
88+
},
89+
{
90+
"use": true,
91+
"Container": {
92+
"disk": 4000,
93+
"mem": 8000,
94+
"name": "log-server",
95+
"cpu": 2
96+
}
97+
},
98+
{
99+
"use": true,
100+
"Container": {
101+
"disk": 4000,
102+
"mem": 8000,
103+
"name": "rabbitmq",
104+
"cpu": 2
105+
}
106+
}
107+
],
108+
"name": "Cluster"
109+
},
110+
"use": true
111+
},
112+
"cd": {
113+
"use": true,
114+
"CD": {
115+
"name": "CD"
116+
}
117+
},
118+
"registry": {
119+
"use": true,
120+
"Registry": {
121+
"name": "Registry"
122+
}
123+
},
124+
"vpc": {
125+
"use": true,
126+
"VPC": {
127+
"name": "VPC"
128+
}
129+
},
130+
"region": "us-east-2"
131+
}
132+
~~~
133+
Purpose of `use`
134+
- Should this part of `Deployment` be used in translating to Cloudformation?
135+
~~~shell
136+
barak@berkos:~/deployment/build (master) $ echo '{"region": "us-east-1", "cluster": {"use": false, "Cluster":{"name": "myCluster", "tasks":[]}}}' | ./result/bin/atidot-deployment translate | jq .Resources.myCluster
137+
null
138+
barak@berkos:~/deployment/build (master) $ echo '{"region": "us-east-1", "cluster": {"use": true, "Cluster":{"name": "myCluster", "tasks":[]}}}' | ./result/bin/atidot-deployment translate | jq .Resources.myCluster
139+
{
140+
"DependsOn": [
141+
"VPC"
142+
],
143+
"Type": "AWS::ECS::Cluster",
144+
"Properties": {}
145+
}
146+
barak@berkos:~/deployment/build (master) $
147+
~~~
148+
# Editor
149+
- http://104.40.92.122/
150+
![](editor.gif)

‎atidot-deployment-editor/dist-newstyle/cache/compiler

11.7 KB
Binary file not shown.

‎atidot-deployment-editor/dist-newstyle/cache/config

5.97 KB
Binary file not shown.

‎atidot-deployment-editor/src/Atidot/Deployment/Frontend.hs

+15-29
Original file line numberDiff line numberDiff line change
@@ -41,29 +41,7 @@ import "reflex-select2" Reflex.Select2.Select2
4141
import "reflex-jexcel" Reflex.JExcel
4242
import "reflex-fileapi" Reflex.FileAPI.FileAPI
4343
import "atidot-deployment" Atidot.Deployment.Types
44-
import "atidot-deployment" Atidot.Deployment.Compile
45-
46-
47-
48-
defaultDeployment :: Deployment
49-
defaultDeployment
50-
= def
51-
& deployment_cluster
52-
.~ ( Yes
53-
$ def
54-
& cluster_tasks
55-
.~ [ Yes $ def
56-
& container_name .~ "undins"
57-
, Yes $ def
58-
& container_name .~ "jobrunner"
59-
, Yes $ def
60-
& container_name .~ "api"
61-
, Yes $ def
62-
& container_name .~ "log-server"
63-
, Yes $ def
64-
& container_name .~ "rabbitmq"
65-
]
66-
)
44+
import "atidot-deployment" Atidot.Deployment.Deployment
6745

6846

6947
main :: IO ()
@@ -114,27 +92,35 @@ body :: forall t m. (MonadWidget t m)
11492
body = mdlGrid $ do
11593
-- initial default configuration
11694
postBuildE <- getPostBuild
117-
let deploymentE = defaultDeployment <$ postBuildE
95+
let deploymentE = exampleDeployment <$ postBuildE
11896
(initialD :: Dynamic t Deployment) <- holdDyn def deploymentE
11997

12098
-- json editor
121-
(userInputE :: Event t Deployment) <- mdlCell 4 $ do
122-
mdlGrid $ el "h3" $ text "Atidot Configuration"
123-
(xD, _) <- jsoneditor def initialD
99+
(userInputE :: Event t Deployment) <- mdlCell 5 $ do
100+
mdlGrid $ el "h3" $ text "Atidot Deployment"
101+
(xD, _) <- jsoneditor jsConfig initialD
124102
display xD
125103
return $ updated xD
126104

127105
-- compile to CloudFormation
128-
let cloudformationE = toCloudFormation <$> userInputE
106+
let cloudformationE = toCloudFormation' <$> userInputE
129107
let cfTextE = (decodeUtf8 . toStrict . encodeTemplate) <$> cloudformationE
130108

131109
-- codemirror
132-
_ <- mdlCell 8 $ do
110+
_ <- mdlCell 7 $ do
133111
mdlGrid $ el "h3" $ text "CloudFormation JSON"
134112
codemirror cmConfig cfTextE never
135113

136114
return ()
137115
where
116+
jsConfig :: JsonEditorOptions
117+
jsConfig
118+
= def
119+
& jsonEditorOptions_mode ?~ Tree
120+
& jsonEditorOptions_modes ?~ [ Tree
121+
, Code
122+
]
123+
138124
cmConfig :: Configuration
139125
cmConfig
140126
= def

‎atidot-deployment/app/Main.hs

+55-8
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,65 @@
22
{-# LANGUAGE ScopedTypeVariables #-}
33
module Main where
44

5-
import "base" Data.Functor.Identity (Identity)
6-
import "text" Data.Text (Text)
5+
import "base" Data.Maybe (fromJust)
76
import "base" System.IO (stdin)
7+
import "text" Data.Text (Text)
88
import qualified "bytestring" Data.ByteString.Lazy.Char8 as B (ByteString, readFile, putStrLn, hGetContents, unpack)
9-
import "vector" Data.Vector (Vector)
10-
import qualified "cassava" Data.Csv as CSV (Header, encodeDefaultOrderedByName, decodeByName)
11-
import qualified "aeson" Data.Aeson as JSON (encode)
12-
import "QuickCheck" Test.QuickCheck.Arbitrary
13-
import "QuickCheck" Test.QuickCheck.Gen
9+
import qualified "aeson" Data.Aeson as JSON (eitherDecode, encode)
10+
import qualified "aeson-pretty" Data.Aeson.Encode.Pretty as JSON (encodePretty)
11+
import "stratosphere" Stratosphere hiding ((.=))
1412
import "optparse-applicative" Options.Applicative
1513
import "atidot-deployment" Atidot.Deployment.Types
14+
import "atidot-deployment" Atidot.Deployment.Deployment
15+
16+
data Example
17+
= Example
18+
deriving (Show)
19+
20+
data Translate
21+
= Translate
22+
deriving (Show)
23+
24+
data Command
25+
= C1 Example
26+
| C2 Translate
27+
deriving (Show)
28+
29+
30+
exampleParser :: Parser Command
31+
exampleParser = pure $ C1 Example
32+
33+
translateParser :: Parser Command
34+
translateParser = pure $ C2 Translate
35+
36+
combined :: Parser Command
37+
combined
38+
= subparser
39+
( command "example" (info exampleParser (progDesc "Print exapmle Deployment"))
40+
<> command "translate" (info translateParser (progDesc "Translate Deployment JSON (<STDIN>) to CloudFormation JSON (<STDOUT>)"))
41+
)
42+
43+
44+
onExample :: Example -> IO ()
45+
onExample _ = do
46+
B.putStrLn . JSON.encodePretty $ exampleDeployment
47+
48+
onTranslate :: Translate -> IO ()
49+
onTranslate _ = do
50+
eDeployment <- JSON.eitherDecode <$> B.hGetContents stdin
51+
case eDeployment of
52+
Left err -> error err
53+
Right deployment -> do
54+
let output = encodeTemplate
55+
. toCloudFormation'
56+
$ deployment
57+
B.putStrLn output
58+
1659

1760
main :: IO ()
1861
main = do
19-
return ()
62+
options <- execParser (info (combined <**> helper)
63+
(fullDesc <> progDesc "Atidot Deployment"))
64+
case options of
65+
C1 example -> onExample example
66+
C2 translate -> onTranslate translate

‎atidot-deployment/atidot-deployment.cabal

+9-1
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,22 @@ library
2727
, Atidot.Deployment.Types.CSV
2828
, Atidot.Deployment.Monad
2929
, Atidot.Deployment.Compile
30+
, Atidot.Deployment.VPC
31+
, Atidot.Deployment.CD
32+
, Atidot.Deployment.Registry
33+
, Atidot.Deployment.Roles
34+
, Atidot.Deployment.Cluster
35+
, Atidot.Deployment.Container
36+
, Atidot.Deployment.Deployment
3037
-- other-modules:
3138
-- other-extensions:
3239
build-depends: base
3340
, lens
34-
, uniplate
3541
, aeson
3642
, data-default
3743
, bytestring
3844
, text
45+
, casing
3946
, vector
4047
, cassava
4148
, transformers
@@ -70,6 +77,7 @@ executable atidot-deployment
7077
, bytestring
7178
, text
7279
, aeson
80+
, aeson-pretty
7381
, cassava
7482
, vector
7583
, QuickCheck

‎atidot-deployment/src/Atidot/Deployment/CD.hs

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{-# LANGUAGE PackageImports #-}
2+
{-# LANGUAGE OverloadedStrings #-}
3+
{-# LANGUAGE OverloadedLists #-}
4+
{-# LANGUAGE TemplateHaskell #-}
5+
{-# LANGUAGE ScopedTypeVariables #-}
6+
{-# LANGUAGE StandaloneDeriving #-}
7+
{-# LANGUAGE DeriveAnyClass #-}
8+
{-# LANGUAGE RecordWildCards #-}
9+
{-# LANGUAGE RankNTypes #-}
10+
11+
module Atidot.Deployment.CD where
12+
13+
import "data-default" Data.Default (def)
14+
import "mtl" Control.Monad.Reader (ask, asks)
15+
import "lens" Data.Data.Lens hiding (template)
16+
import "aeson" Data.Aeson
17+
import "text" Data.Text (Text, pack)
18+
import "stratosphere" Stratosphere hiding ((.=))
19+
import Atidot.Deployment.Types
20+
import Atidot.Deployment.Monad
21+
import Atidot.Deployment.Compile
22+
23+
instance ToResources CD where
24+
toResources _
25+
= return
26+
[ resource "CD" ecsCluster
27+
]

0 commit comments

Comments
 (0)