diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml new file mode 100644 index 00000000..3526291a --- /dev/null +++ b/.github/auto_assign.yml @@ -0,0 +1,14 @@ +# Set to true to add reviewers to pull requests +addReviewers: true + +# Set to true to add assignees to pull requests +addAssignees: false + +# A list of reviewers to be added to pull requests (GitHub user name) +reviewers: + - adbharadwaj # Aditya Bharadwaj +#(optional) - tmmurali + +# A number of reviewers added to the pull request +# Set 0 to add all the reviewers (default: 0) +numberOfReviewers: 0 diff --git a/.github/pr-labeler.yml b/.github/pr-labeler.yml new file mode 100644 index 00000000..e8e2a265 --- /dev/null +++ b/.github/pr-labeler.yml @@ -0,0 +1,4 @@ +feature: ['feature/*', 'feat/*'] +fix: fix/* +chore: chore/* +actions-pipeline: actions-pipeline/* \ No newline at end of file diff --git a/.github/workflows/Image_scan.yaml b/.github/workflows/Image_scan.yaml new file mode 100644 index 00000000..ce7d6ed7 --- /dev/null +++ b/.github/workflows/Image_scan.yaml @@ -0,0 +1,24 @@ +name: Trivy Docker Image Scan + +on: [push, pull_request] # Runs when the someone pushes changes to branch or creates a pull request + +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build an image from Dockerfile + run: docker build -t docker.io/my-organization/my-app:${{ github.sha }} . + + - name: Run Trivy vulnerability scanner # Runs the Trivy Vulnerability Scanner and Prints the results + uses: aquasecurity/trivy-action@master + with: + image-ref: 'docker.io/my-organization/my-app:${{ github.sha }}' + format: 'table' + exit-code: '0' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' \ No newline at end of file diff --git a/.github/workflows/assign_pr.yaml b/.github/workflows/assign_pr.yaml new file mode 100644 index 00000000..8d37872b --- /dev/null +++ b/.github/workflows/assign_pr.yaml @@ -0,0 +1,13 @@ +name: 'Auto Assign' +on: + pull_request_target: + types: [opened, ready_for_review] # Runs when the someone opens a pull request + +jobs: + add-reviews: + runs-on: ubuntu-latest + steps: + - name: Auto Assign Reviewer # Assigns Reviwer from reviewers list to the Pull Request + uses: kentaro-m/auto-assign-action@v1.2.5 + with: + configuration-path: '.github/auto_assign.yml' # Only needed if you use something other than .github/auto_assign.yml \ No newline at end of file diff --git a/.github/workflows/ci-graphspace.yaml b/.github/workflows/ci-graphspace.yaml new file mode 100644 index 00000000..cae200b2 --- /dev/null +++ b/.github/workflows/ci-graphspace.yaml @@ -0,0 +1,32 @@ +name: Docker Build & Push Graphspace + +on: [push, pull_request] # Runs when the someone pushes changes to branch or creates a pull request + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: login to docker hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Set Tag + id: vars + if: ${{ github.ref == 'refs/heads/master' }} + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Create Graphspace image + run: docker build -t graphspace . + + - name: Push image to Dockerhub + if: ${{ github.ref == 'refs/heads/master' }} + run: | + docker tag graphspace maheshkasbe/graphspace:${{ steps.vars.outputs.sha_short }} + docker tag graphspace maheshkasbe/graphspace:latest + docker push maheshkasbe/graphspace:latest + docker push maheshkasbe/graphspace:${{ steps.vars.outputs.sha_short }} \ No newline at end of file diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 00000000..665cfe73 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,48 @@ +name: Push to ECR + +on: [push, pull_request] # Runs when the someone pushes changes to branch or creates a pull request + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + + - name: Check out code + uses: actions/checkout@v2 + + - name: Build the Docker-compose file + run: docker-compose -f "docker-compose.yaml" up -d --build + + - name: Show containers + run: docker ps -a + + - name: Configure AWS credentials + if: ${{ github.ref == 'refs/heads/master' }} + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Login to Amazon ECR + if: ${{ github.ref == 'refs/heads/master' }} + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + registry-type: public + + - name: Set Tag + id: vars + if: ${{ github.ref == 'refs/heads/master' }} + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Build, tag, and push image to Amazon ECR + if: ${{ github.ref == 'refs/heads/master' }} + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REGISTRY_ALIAS: ${{ secrets.AWS_REGISTRY_ALIAS }} + ECR_REPOSITORY: ${{ secrets.AWS_ECR_REPOSITORY }} + IMAGE_TAG: ${{ steps.vars.outputs.sha_short }} + run: | + docker build -t $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file diff --git a/.github/workflows/issue_label.yaml b/.github/workflows/issue_label.yaml new file mode 100644 index 00000000..2d66a45b --- /dev/null +++ b/.github/workflows/issue_label.yaml @@ -0,0 +1,15 @@ +name: Labeling new issue +on: + issues: + types: ['opened','edited'] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Issue Label #Labels the issue by the description of the issue + uses: Renato66/auto-label@v2 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + ignore-comments: true + labels-synonyms: '{"bug":["error","need fix","not working"],"enhancement":["upgrade"],"question":["help"]}' + labels-not-allowed: '["good first issue"]' \ No newline at end of file diff --git a/.github/workflows/pr_labeler.yaml b/.github/workflows/pr_labeler.yaml new file mode 100644 index 00000000..6f533631 --- /dev/null +++ b/.github/workflows/pr_labeler.yaml @@ -0,0 +1,32 @@ +name: PR Labeler +on: pull_request_target # Runs when the someone creates a pull request + +permissions: + contents: read + pull-requests: write + +jobs: + pr-labeler: + runs-on: ubuntu-latest + + steps: + - name: branch-label # Labels the Pull Request by branch name as which branch the pr is raised to + uses: TimonVS/pr-labeler-action@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/pr-labeler.yml # optional, .github/pr-labeler.yml is the default value + + - name: size-label # Labels the Pull Request by the no of lines of code + uses: "pascalgn/size-label-action@v0.4.2" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + sizes: > + { + "0": "XS", + "20": "S", + "50": "M", + "200": "L", + "800": "XL", + "2000": "XXL" + } \ No newline at end of file diff --git a/.gitignore b/.gitignore index ee495b76..20f65f20 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ # Ignore updates to production.py, # It's meant to be local -production.py # Ignore Mac DS_Store files .DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f17aee03 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +#Base Image to install & Run Graphspace +FROM ubuntu:18.04 + +#Created a app directory for graphspace +WORKDIR /app + +#Installs required & dev packages to smoothly install graphspace rquirements on ubuntu and mac as well +RUN apt-get -y update && apt-get install -y \ + npm \ + git \ + libpq-dev \ + libxml2 \ + libxslt-dev + +#Installs npm globally +RUN npm install bower -g + +#Installs Pip package and upgrades it +RUN apt-get install -y python-pip && pip install --upgrade pip + +#Copies whole graphspace project in app directory +COPY . /app + +#Installs Graphspace requirements +RUN pip install -r requirements.txt + +#Installs Graphspace +RUN sh install.sh + +#Exposes the port 8000 to discover graphspace container +EXPOSE 8000 \ No newline at end of file diff --git a/README.md b/README.md index 22ffebc1..418a22f9 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Requirements Installation Instructions =================================== * [Running GraphSpace Locally](https://github.com/Murali-group/GraphSpace/wiki/Running-GraphSpace-locally) +* [Running Graphspace Locally using Docker & Docker‐Compose](https://github.com/Murali-group/GraphSpace/wiki/Running-Graphspace-using-Docker-&-Docker‐Compose) * [Running GraphSpace on Apache](https://github.com/Murali-group/GraphSpace/wiki/Running-GraphSpace-on-Apache) Documentation diff --git a/bower.json b/bower.json index 7b986c01..d2134384 100644 --- a/bower.json +++ b/bower.json @@ -23,11 +23,9 @@ "animate.css": "^3.5.2", "cytoscape": "^2.7.11", "webcola": "^3.3.0", - "bootstrap": "^3.3.7", "cytoscape-cola": "^1.6.0", "intro.js": "^2.4.0", "jquery-ui": "^1.12.1", - "bootstrap-table": "^1.11.0", "cytoscape-panzoom": "^2.4.0", "select2": "select2-dist#^4.0.3", "cytoscape-context-menus": "^2.1.1", diff --git a/db.sh b/db.sh new file mode 100755 index 00000000..c2b64057 --- /dev/null +++ b/db.sh @@ -0,0 +1,9 @@ +#!bin/sh + +#connects to the postgres database & creates pg_trgm & btree_gin extensions +echo "enabling pg_trgm & btree_gin on database $POSTGRES_DB" +psql -U $POSTGRES_USER --dbname="$POSTGRES_DB" <<-'EOSQL' +create extension if not exists pg_trgm; +create extension if not exists btree_gin; +EOSQL +echo "finished with exit code $?" \ No newline at end of file diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml new file mode 100644 index 00000000..ca566632 --- /dev/null +++ b/docker-compose.prod.yaml @@ -0,0 +1,15 @@ +version: "3.4" +services: + graphspace: + restart: always + stdin_open: true # docker run -i + tty: true # docker run -t + build: + context: . + ports: + - "8000:8000" + env_file: + - ./secrets.env + command: > + sh -c "python manage.py migrate && + python manage.py runserver" \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..05bac910 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,52 @@ +version: "3.4" +services: +#Postgres container with env variables + db: + image: postgres:9.6 + restart: always + environment: + POSTGRES_DB: graphspace + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/PostgreSQL/data + - ./db.sh:/docker-entrypoint-initdb.d/create_extensions.sh + +#Elasticsearch Container + elasticsearch: + restart: always + image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2 + ports: + - "9200:9200" + environment: + - discovery.type=single-node + volumes: + - esdata:/var/lib/elasticsearch/data + +#Graphspace Image with enviornment variables + graphspace: + restart: always + image: maheshkasbe/graphspace:latest + environment: + POSTGRES_HOST: db + POSTGRES_DB: graphspace + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + stdin_open: true # docker run -i + tty: true # docker run -t + # build: + # context: . + ports: + - "8000:8000" + command: > + sh -c "python manage.py migrate --settings=graphspace.settings.local && + python manage.py runserver 0.0.0.0:8000 --settings=graphspace.settings.local" + depends_on: + - db + - elasticsearch + +volumes: + pgdata: + esdata: \ No newline at end of file diff --git a/graphspace/settings/base.py b/graphspace/settings/base.py index f0bf095c..980f3fca 100644 --- a/graphspace/settings/base.py +++ b/graphspace/settings/base.py @@ -140,7 +140,8 @@ ) BASE = declarative_base() -ELASTIC_CLIENT = Elasticsearch() +# for connecting with elasticsearch client using hostname and port +ELASTIC_CLIENT = Elasticsearch(['elasticsearch:9200']) LOGGING = { 'version': 1, diff --git a/graphspace/settings/local.py b/graphspace/settings/local.py index b15da20d..be5f1ad1 100644 --- a/graphspace/settings/local.py +++ b/graphspace/settings/local.py @@ -36,14 +36,14 @@ # http://stackoverflow.com/questions/4642011/test-sending-email-without-email-server EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' - +# Added enviornment variable option to run with doccker-compose file and also with local dev setup DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'test_database', - 'USER': 'adb', - 'PASSWORD': '', - 'HOST': 'localhost', + 'NAME': os.environ.get('POSTGRES_DB', 'graphspace'), + 'USER': os.environ.get('POSTGRES_USER', 'postgres'), + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD', 'postgres'), + 'HOST': os.environ.get('POSTGRES_HOST', 'localhost'), 'PORT': '5432' } } diff --git a/graphspace/settings/production.py b/graphspace/settings/production.py new file mode 100644 index 00000000..23b7fa28 --- /dev/null +++ b/graphspace/settings/production.py @@ -0,0 +1,49 @@ +from graphspace.settings.base import * + +# variables for setting up account through which GraphSpace emails +EMAIL_HOST = os.environ.get('EMAIL_HOST') +EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') + +# If error is thrown, display the error in the browser (ONLY FOR LOCAL MACHINES) +DEBUG = os.environ.get('DEBUG') +TEMPLATE_DEBUG = os.environ.get('TEMPLATE_DEBUG') +MAINTENANCE = os.environ.get('MAINTENANCE') + +# URL through which to access graphspace +URL_PATH = os.environ.get('URL_PATH') + +# If tracking is enabled for GraphSpace in Google Analytics +GOOGLE_ANALYTICS_PROPERTY_ID = os.environ.get('GOOGLE_ANALYTICS_PROPERTY_ID') + +# Keys given by creating a requestor account on Amazon Mechanical Turk (https://www.mturk.com/mturk/welcome) +AWSACCESSKEYID = os.environ.get('AWSACCESSKEYID') +SECRETKEY = os.environ.get('SECRETKEYOST') + +# Path to GraphSPace +PATH = os.environ.get('PATH', '/path_to_graphspace') + +# SHOULD NEVER CHANGE THIS VALUE +SECRET_KEY = os.environ.get('SECRET_KEY') + +# If needing to test on production mturk account (real money) +# AWS_URL = 'https://mechanicalturk.amazonaws.com' + +# Sandbox (development) MTURK (fake money used) +AWS_URL = os.environ.get('AWS_URL') + +# To configure the application to use the Console Backend for sending e-mail. It writes e-mails to standard out instead of sending them. +# http://stackoverflow.com/questions/4642011/test-sending-email-without-email-server +EMAIL_BACKEND = os.environ.get('EMAIL_BACKEND') + + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.environ.get('POSTGRES_DB'), + 'USER': os.environ.get('POSTGRES_USER'), + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD'), + 'HOST': os.environ.get('POSTGRES_HOST'), + 'PORT': os.environ.get('POSTGRES_PORT') + } +} diff --git a/install.sh b/install.sh index c6a19f4b..72d9858b 100644 --- a/install.sh +++ b/install.sh @@ -3,3 +3,6 @@ python setup.py install python manage.py migrate --settings=graphspace.settings.local bower install +#Temporary & forced fix untill all the libraries get updated +bower install bootstrap#3.3.7 +bower install bootstrap-table#1.11.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 36f14a9e..8c03026e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ MarkupSafe==0.23 networkx==1.11 oauthlib==1.1.2 poster==0.8.1 -psycopg2==2.6.2 +psycopg2==2.7.4 py-bcrypt==0.4 Pygments==2.5.2 pytz==2016.4 @@ -29,3 +29,5 @@ sphinx-rtd-theme sphinx recommonmark python-dotenv==0.15.0 +lxml +psycopg2-binary \ No newline at end of file diff --git a/secrets.env b/secrets.env new file mode 100644 index 00000000..14abd917 --- /dev/null +++ b/secrets.env @@ -0,0 +1,18 @@ +EMAIL_HOST=None +EMAIL_HOST_USER=None +EMAIL_HOST_PASSWORD=None +DEBUG=False +TEMPLATE_DEBUG=True +MAINTENANCE=False +URL_PATH=http://localhost=8000/ +GOOGLE_ANALYTICS_PROPERTY_ID=UA-00000000-0 +AWSACCESSKEYID=None +SECRETKEY=None +SECRET_KEY=this-is-a-secret-key-for-local-settings-only +AWS_URL=https://mechanicalturk.sandbox.amazonaws.com +EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend +POSTGRES_DB=graphspace +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_HOST=db +POSTGRES_PORT=5432 \ No newline at end of file