Skip to content

Commit bac1c92

Browse files
arthuralvimarthuralvim
committed
Subindo aplicação inicial.
1 parent 4313ac7 commit bac1c92

22 files changed

+1084
-0
lines changed

‎.coveragerc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[run]
2+
source = .
3+
omit = tests/*

‎.dockerignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
.*
2+
!.coveragerc
3+
!.env
4+
!.pylintrc
5+
Dockerfile
6+
README.md

‎.env-example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
COVERALLS_SERVICE_NAME=travis-pro
2+
COVERALLS_REPO_TOKEN=
3+
DOCKER_REGISTRY=

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,7 @@ venv.bak/
102102

103103
# mypy
104104
.mypy_cache/
105+
106+
# tests
107+
tests/media/*
108+
.DS_Store

‎.pre-commit-config.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
fail_fast: true
2+
repos:
3+
- repo: git://github.com/pre-commit/pre-commit-hooks
4+
rev: v2.2.1
5+
hooks:
6+
- id: check-added-large-files
7+
- id: check-merge-conflict
8+
- id: check-symlinks
9+
- id: check-yaml
10+
- id: debug-statements
11+
- id: detect-private-key
12+
- id: double-quote-string-fixer
13+
- id: end-of-file-fixer
14+
- id: flake8
15+
- id: no-commit-to-branch

‎.travis.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# travis-ci jurimetria
2+
3+
sudo: required
4+
5+
language: python
6+
7+
python:
8+
- 3.7.0
9+
10+
services:
11+
- docker
12+
13+
before_install:
14+
- echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$PR, BRANCH=$BRANCH, TRAVIS_TAG=$TRAVIS_TAG"
15+
- pip install --upgrade pip==19.1.1
16+
- pip install pipenv
17+
- sudo rm -f /etc/boto.cfg
18+
- pipenv install --system --dev
19+
- make pep8
20+
21+
install:
22+
- echo "nothing else to install!"
23+
24+
script:
25+
- make test
26+
27+
after_script:
28+
- set -e
29+
- coveralls
30+
- docker --version
31+
- export PATH=$PATH:$HOME/.local/bin # put aws in the path
32+
- make docker.login
33+
- make docker.build
34+
- make docker.push
35+
36+
after_failure:
37+
- if [ -f log/tests.log ]; then cat log/tests.log && sleep 5; fi
38+
- echo "== End of CI Log ==""
39+
40+
cache:
41+
directories:
42+
- $HOME/.cache/.pip
43+
- $HOME/.cache/.pipenv
44+
45+
notifications:
46+
email: false

‎Dockerfile

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
FROM amazonlinux:2
2+
MAINTAINER arthuralvim
3+
4+
ENV WORKDIR=/var/task \
5+
PIPENV_VENV_IN_PROJECT=1 \
6+
PYTHON_VERSION=3.7.0 \
7+
PIP_VERSION=19.3.1 \
8+
LC_ALL=C.UTF-8 \
9+
LANG=C.UTF-8 \
10+
S3_INPUT=s3://bucket-example \
11+
DEBUG=True
12+
13+
ENV PYENV_ROOT $WORKDIR/.pyenv
14+
ENV PIPENV_CACHE_DIR $WORKDIR/.pipenv
15+
ENV PIPENV_PYTHON ${PYENV_ROOT}/shims/python
16+
ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
17+
ENV BUILD_PACKAGES bzip2-devel gcc git wget which libxml2-dev libxslt-dev make \
18+
openssl-devel python36-dev readline-devel postgresql-devel \
19+
libffi-devel sqlite-devel tar
20+
21+
WORKDIR ${WORKDIR}
22+
COPY . "$WORKDIR"
23+
24+
RUN yum install -y ${BUILD_PACKAGES} && \
25+
git clone git://github.com/yyuu/pyenv.git .pyenv && \
26+
pyenv install ${PYTHON_VERSION} && \
27+
pyenv global ${PYTHON_VERSION} && \
28+
pyenv rehash && \
29+
pip install --upgrade pip==${PIP_VERSION} && \
30+
pip install pipenv --no-cache-dir && \
31+
pipenv install --deploy && \
32+
yum clean all && \
33+
rm -rf /var/cache/yum && \
34+
rm -rf ${PIPENV_CACHE_DIR}
35+
36+
RUN if test "$(S3_INPUT)" = "" ; then echo "S3_INPUT is undefined."; else aws s3 cp ${S3_INPUT} ${WORKDIR}/input/; fi
37+
38+
CMD ["pipenv", "run", "python", "run.py"]

‎Makefile

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Makefile batch-data-engineering
2+
3+
# These targets are not files
4+
.PHONY: all help check.test_path requirements clean pep8 test test.dev test.failfirst test.collect test.skip.covered coverage coverage.html coveralls docker.login docker.build_ docker.tag docker.build docker.push
5+
6+
all: help
7+
8+
help:
9+
@echo 'Makefile *** batch-data-engineering *** Makefile'
10+
11+
check.test_path:
12+
@if test "$(TEST_PATH)" = "" ; then echo "TEST_PATH is undefined. The default is tests."; fi
13+
14+
requirements:
15+
@pipenv lock --requirements > requirements.txt
16+
17+
clean:
18+
@find . -name '*.pyc' -exec rm -f {} \;
19+
@find . -name 'Thumbs.db' -exec rm -f {} \;
20+
@find . -name '*~' -exec rm -f {} \;
21+
22+
pep8:
23+
@pycodestyle --filename="*.py" .
24+
25+
### TESTS
26+
27+
test: check.test_path
28+
@py.test -s $(TEST_PATH) --cov --cov-report term-missing --basetemp=tests/media --disable-pytest-warnings
29+
30+
test.dev: check.test_path
31+
@py.test -s $(TEST_PATH) --cov --cov-fail-under 70 --cov-report term-missing --basetemp=tests/media --disable-pytest-warnings
32+
33+
test.failfirst: check.test_path
34+
@py.test -s -x $(TEST_PATH) --basetemp=tests/media --disable-pytest-warnings
35+
36+
test.collect: check.test_path
37+
@py.test -s $(TEST_PATH) --basetemp=tests/media --collect-only --disable-pytest-warnings
38+
39+
test.skip.covered: check.test_path
40+
@py.test -s $(TEST_PATH) --cov --cov-report term:skip-covered --doctest-modules --basetemp=tests/media --disable-pytest-warnings
41+
42+
coverage: check.test_path test
43+
44+
coverage.html: check.test_path
45+
@py.test -s $(TEST_PATH) --cov --cov-report html --doctest-modules --basetemp=tests/media --disable-pytest-warnings
46+
47+
coveralls: coverage
48+
@coveralls
49+
50+
### DOCKER
51+
DOCKER_REGISTRY := 296022280050.dkr.ecr.us-east-1.amazonaws.com
52+
DOCKER_NAME := arthuralvim/tutorial-batch-data-engineering
53+
DOCKER_TAG := $$(if [ "${TRAVIS_TAG}" = "" ]; then echo `git log -1 --pretty=%h`; else echo "${TRAVIS_TAG}"; fi)
54+
DOCKER_IMG_TAG := ${DOCKER_NAME}:${DOCKER_TAG}
55+
DOCKER_LATEST := ${DOCKER_NAME}:latest
56+
DOCKER_PR_BRANCH := ${DOCKER_NAME}:${TRAVIS_PULL_REQUEST_BRANCH}
57+
58+
check.docker_registry:
59+
@if test "$(DOCKER_REGISTRY)" = "" ; then echo "DOCKER_REGISTRY is undefined."; exit 1; fi
60+
61+
docker.login:
62+
$$(aws ecr get-login --no-include-email --region us-east-1)
63+
64+
docker.build_: check.docker_registry
65+
@echo "Build started on `date`"
66+
@docker build -f Dockerfile -t ${DOCKER_IMG_TAG} .
67+
@echo "Build completed on `date`"
68+
69+
docker.tag: check.docker_registry
70+
@if [ ! -z "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then docker tag ${DOCKER_IMG_TAG} ${DOCKER_REGISTRY}/${DOCKER_PR_BRANCH}; fi
71+
@docker tag ${DOCKER_IMG_TAG} ${DOCKER_REGISTRY}/${DOCKER_LATEST}
72+
@docker tag ${DOCKER_IMG_TAG} ${DOCKER_REGISTRY}/${DOCKER_IMG_TAG}
73+
74+
docker.build: docker.build_ docker.tag
75+
76+
docker.push: check.docker_registry
77+
@echo "Pushing images started on `date`"
78+
@if [ ! -z "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then docker push ${DOCKER_REGISTRY}/${DOCKER_PR_BRANCH}; fi
79+
@docker push ${DOCKER_REGISTRY}/${DOCKER_LATEST}
80+
@docker push ${DOCKER_REGISTRY}/${DOCKER_IMG_TAG}
81+
@echo "Pushing images completed on `date`"

‎Pipfile

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
[[source]]
2+
name = "pypi"
3+
url = "https://pypi.org/simple"
4+
verify_ssl = true
5+
6+
[dev-packages]
7+
coverage = "*"
8+
coveralls = "*"
9+
ipdb = "*"
10+
ipython = "==7.0.0"
11+
jupyter = "*"
12+
pep8 = "*"
13+
pre-commit = "*"
14+
pycodestyle = "*"
15+
pytest = "*"
16+
pytest-cov = "*"
17+
pytest-mock = "*"
18+
pytest-sugar = "*"
19+
20+
[packages]
21+
awscli = "*"
22+
boto3 = "*"
23+
python-decouple = "*"
24+
watchtower = "*"
25+
26+
[requires]
27+
python_version = "3.7.0"

0 commit comments

Comments
 (0)