diff --git a/.github/workflows/ci-simple.yml b/.github/workflows/ci-simple.yml deleted file mode 100644 index 1413967..0000000 --- a/.github/workflows/ci-simple.yml +++ /dev/null @@ -1,50 +0,0 @@ -# GitHub Actions CI Workflow for StockSense Agent -# Automated testing pipeline for Python application - -name: Python Application CI - -# Trigger the workflow on push and pull requests to main branch -on: - push: - branches: [main] - pull_request: - branches: [main] - -# Define the CI job -jobs: - build: - # Job name and runner configuration - name: Build and Test - runs-on: ubuntu-latest - - steps: - # Step 1: Checkout code from repository - - name: Checkout code - uses: actions/checkout@v4 - - # Step 2: Set up Python 3.10 environment - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - # Step 3: Install dependencies - - name: Upgrade pip - run: | - python -m pip install --upgrade pip - - - name: Install project dependencies - run: | - pip install -r requirements.txt - - - name: Install testing dependencies - run: | - pip install pytest requests - - # Step 4: Run tests with environment variables - - name: Run tests - run: | - pytest tests/ -v - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cfe621..dd87a63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,126 +1,21 @@ # GitHub Actions CI Workflow for StockSense Agent -# This workflow runs automated tests on every push and pull request to main branch +# Validates code, tests, and Docker builds on every push/pull_request to main. -name: Python Application CI +name: CI - StockSense Agent -# Trigger Configuration -# Runs on pushes to main branch and all pull requests targeting main on: push: branches: [main] pull_request: branches: [main] -# Define CI Jobs jobs: - build: - # Job Configuration - name: Build and Test + test-and-validate: + name: Run Tests and Validate Docker Images runs-on: ubuntu-latest - # Define build steps steps: - # Step 1: Checkout Repository Code - - name: Checkout code - uses: actions/checkout@v4 - with: - # Fetch full history for better Git operations - fetch-depth: 0 - - # Step 2: Set up Python Environment - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: '3.10' - # Cache pip dependencies for faster builds - cache: 'pip' - - # Step 3: Install Dependencies - - name: Upgrade pip - run: | - python -m pip install --upgrade pip - - - name: Install project dependencies - run: | - pip install -r requirements.txt - - - name: Install testing dependencies - run: | - pip install pytest requests - - # Step 4: Run Linting (Optional but recommended) - - name: Run code quality checks - run: | - # Install flake8 for linting - pip install flake8 - # Stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # Exit-zero treats all errors as warnings. GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - continue-on-error: true - - # Step 5: Run Unit Tests (No external dependencies required) - - name: Run unit tests - run: | - # Run only unit tests that don't require external services - python run_tests.py unit - env: - # Set environment variables from GitHub secrets - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }} - - # Step 6: Run Smoke Tests (Quick validation) - - name: Run smoke tests - run: | - # Run quick smoke tests for basic functionality validation - python run_tests.py smoke - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }} - - # Step 7: Run Full Test Suite with Coverage (Optional) - - name: Run tests with coverage - run: | - # Install coverage tool - pip install coverage - # Run pytest with coverage reporting - coverage run -m pytest tests/test_tools.py -v - # Generate coverage report - coverage report -m - # Generate HTML coverage report - coverage html - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }} - continue-on-error: true - - # Step 8: Upload Coverage Reports (Optional) - - name: Upload coverage reports - uses: actions/upload-artifact@v4 - with: - name: coverage-report - path: htmlcov/ - if: always() - - # Step 9: Upload Test Results - - name: Upload test results - uses: actions/upload-artifact@v4 - with: - name: test-results - path: | - .pytest_cache/ - test-results.xml - if: always() - - # Additional Jobs for Different Scenarios - integration-tests: - # Only run integration tests if unit tests pass - needs: build - name: Integration Tests - runs-on: ubuntu-latest - - steps: - - name: Checkout code + - name: Checkout Repository uses: actions/checkout@v4 - name: Set up Python 3.10 @@ -129,66 +24,24 @@ jobs: python-version: '3.10' cache: 'pip' - - name: Install dependencies + - name: Install All Dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest requests + pip install -r requirements-backend.txt + pip install -r requirements-frontend.txt + pip install pytest pytest-cov - # Start the API server in background for integration tests - - name: Start API server - run: | - # Start the StockSense API server in background - python -m stocksense.main & - # Wait for server to start - sleep 10 + - name: Run Test Suite env: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }} + run: pytest - - name: Run integration tests - run: | - # Run API integration tests - python run_tests.py api - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - # Security and Code Quality Job - security-scan: - name: Security Scan - runs-on: ubuntu-latest + - name: Validate Backend Docker Image + run: docker build -f Dockerfile.backend -t stocksense-backend-ci . - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install security tools - run: | - python -m pip install --upgrade pip - pip install safety bandit - - - name: Run safety check - run: | - # Check for known security vulnerabilities in dependencies - safety check --file requirements.txt - continue-on-error: true - - - name: Run bandit security scan - run: | - # Run security analysis on Python code - bandit -r stocksense/ -f json -o bandit-report.json - continue-on-error: true - - - name: Upload security reports - uses: actions/upload-artifact@v4 - with: - name: security-reports - path: | - bandit-report.json - if: always() + - name: Validate Frontend Docker Image + run: docker build -f Dockerfile.frontend -t stocksense-frontend-ci . \ No newline at end of file diff --git a/.gitignore b/.gitignore index 16cccec..c28d64d 100644 --- a/.gitignore +++ b/.gitignore @@ -78,7 +78,11 @@ env/ /venv/ .conda/ deploy_test_env/ -docker-compose.yml +docker-compose.override.yml + +# Local database (explicit) +stocksense.db +*.db # ============================================================================= # -- Editor & IDE Files -- diff --git a/Dockerfile.backend b/Dockerfile.backend index 542784e..ded6d0c 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -4,18 +4,13 @@ FROM python:3.10-slim # Set working directory WORKDIR /app -# Install system dependencies -RUN apt-get update && apt-get install -y \ - gcc \ - g++ \ - curl \ - && rm -rf /var/lib/apt/lists/* +# No system packages needed; keep image minimal to reduce vulnerabilities # Copy requirements first (for better caching) -COPY requirements.txt . +COPY requirements-backend.txt . # Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements-backend.txt # Copy application code COPY . . @@ -26,9 +21,9 @@ RUN mkdir -p /app/data # Expose port for FastAPI backend EXPOSE 8000 -# Health check for backend API +# Health check for backend API (no curl dependency) HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8000/health || exit 1 + CMD python -c "import sys, requests; sys.exit(0) if requests.get('http://localhost:8000/health', timeout=5).ok else sys.exit(1)" # Run the FastAPI backend application CMD ["uvicorn", "stocksense.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/Dockerfile.frontend b/Dockerfile.frontend index c9829a8..6e938af 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -4,18 +4,13 @@ FROM python:3.10-slim # Set working directory WORKDIR /app -# Install system dependencies -RUN apt-get update && apt-get install -y \ - gcc \ - g++ \ - curl \ - && rm -rf /var/lib/apt/lists/* +# No system packages needed; keep image minimal to reduce vulnerabilities # Copy requirements first (for better caching) -COPY requirements.txt . +COPY requirements-frontend.txt . # Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements-frontend.txt # Copy application code COPY . . @@ -33,9 +28,9 @@ RUN echo '[server]' > ~/.streamlit/config.toml && \ echo 'enableCORS = false' >> ~/.streamlit/config.toml && \ echo 'enableXsrfProtection = false' >> ~/.streamlit/config.toml -# Health check +# Health check (no curl dependency) HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8501/_stcore/health || exit 1 + CMD python -c "import sys, requests; sys.exit(0) if requests.get('http://localhost:8501/_stcore/health', timeout=5).ok else sys.exit(1)" # Run Streamlit app CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e59f8ab --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,75 @@ +# Docker Compose for StockSense Agent +# Runs both frontend and backend services together + +services: + # FastAPI Backend Service + api: + build: + context: . + dockerfile: Dockerfile.backend + container_name: stocksense-api + ports: + - '8000:8000' + environment: + - GOOGLE_API_KEY=${GOOGLE_API_KEY} + - NEWSAPI_KEY=${NEWSAPI_KEY} + - DATABASE_URL=sqlite:///data/stocksense.db + volumes: + - api_data:/app/data + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:8000/health'] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - stocksense-network + + # Streamlit Frontend Service + frontend: + build: + context: . + dockerfile: Dockerfile.frontend + container_name: stocksense-frontend + ports: + - '8501:8501' + environment: + - GOOGLE_API_KEY=${GOOGLE_API_KEY} + - NEWSAPI_KEY=${NEWSAPI_KEY} + - API_BASE_URL=http://api:8000 + depends_on: + api: + condition: service_healthy + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:8501/_stcore/health'] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - stocksense-network + + # Nginx Reverse Proxy (Optional) + nginx: + image: nginx:alpine + container_name: stocksense-proxy + ports: + - '80:80' + - '443:443' + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - frontend + - api + networks: + - stocksense-network + +# Named volumes for data persistence +volumes: + api_data: + driver: local + +# Custom network for service communication +networks: + stocksense-network: + driver: bridge diff --git a/requirements-backend.in b/requirements-backend.in new file mode 100644 index 0000000..be51a72 --- /dev/null +++ b/requirements-backend.in @@ -0,0 +1,10 @@ +fastapi +uvicorn +langgraph +langchain +langchain-google-genai +python-dotenv +yfinance +newsapi-python +requests +pytest \ No newline at end of file diff --git a/requirements-backend.txt b/requirements-backend.txt new file mode 100644 index 0000000..2dd2755 --- /dev/null +++ b/requirements-backend.txt @@ -0,0 +1,241 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements-backend.txt requirements-backend.in +# +annotated-types==0.7.0 + # via pydantic +anyio==4.10.0 + # via + # httpx + # starlette +async-timeout==4.0.3 + # via langchain +beautifulsoup4==4.13.5 + # via yfinance +cachetools==5.5.2 + # via google-auth +certifi==2025.8.3 + # via + # curl-cffi + # httpcore + # httpx + # requests +cffi==1.17.1 + # via curl-cffi +charset-normalizer==3.4.3 + # via requests +click==8.2.1 + # via uvicorn +curl-cffi==0.13.0 + # via yfinance +exceptiongroup==1.3.0 + # via + # anyio + # pytest +fastapi==0.116.1 + # via -r requirements-backend.in +filetype==1.2.0 + # via langchain-google-genai +frozendict==2.4.6 + # via yfinance +google-ai-generativelanguage==0.6.18 + # via langchain-google-genai +google-api-core[grpc]==2.25.1 + # via google-ai-generativelanguage +google-auth==2.40.3 + # via + # google-ai-generativelanguage + # google-api-core +googleapis-common-protos==1.70.0 + # via + # google-api-core + # grpcio-status +grpcio==1.74.0 + # via + # google-api-core + # grpcio-status +grpcio-status==1.74.0 + # via google-api-core +h11==0.16.0 + # via + # httpcore + # uvicorn +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # langgraph-sdk + # langsmith +idna==3.10 + # via + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +langchain==0.3.27 + # via -r requirements-backend.in +langchain-core==0.3.75 + # via + # langchain + # langchain-google-genai + # langchain-text-splitters + # langgraph + # langgraph-checkpoint + # langgraph-prebuilt +langchain-google-genai==2.1.10 + # via -r requirements-backend.in +langchain-text-splitters==0.3.10 + # via langchain +langgraph==0.6.6 + # via -r requirements-backend.in +langgraph-checkpoint==2.1.1 + # via + # langgraph + # langgraph-prebuilt +langgraph-prebuilt==0.6.4 + # via langgraph +langgraph-sdk==0.2.4 + # via langgraph +langsmith==0.4.20 + # via + # langchain + # langchain-core +multitasking==0.0.12 + # via yfinance +newsapi-python==0.2.7 + # via -r requirements-backend.in +numpy==2.2.6 + # via + # pandas + # yfinance +orjson==3.11.3 + # via + # langgraph-sdk + # langsmith +ormsgpack==1.10.0 + # via langgraph-checkpoint +packaging==25.0 + # via + # langchain-core + # langsmith + # pytest +pandas==2.3.2 + # via yfinance +peewee==3.18.2 + # via yfinance +platformdirs==4.4.0 + # via yfinance +pluggy==1.6.0 + # via pytest +proto-plus==1.26.1 + # via + # google-ai-generativelanguage + # google-api-core +protobuf==6.32.0 + # via + # google-ai-generativelanguage + # google-api-core + # googleapis-common-protos + # grpcio-status + # proto-plus + # yfinance +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycparser==2.22 + # via cffi +pydantic==2.11.7 + # via + # fastapi + # langchain + # langchain-core + # langchain-google-genai + # langgraph + # langsmith +pydantic-core==2.33.2 + # via pydantic +pygments==2.19.2 + # via pytest +pytest==8.4.1 + # via -r requirements-backend.in +python-dateutil==2.9.0.post0 + # via pandas +python-dotenv==1.1.1 + # via -r requirements-backend.in +pytz==2025.2 + # via + # pandas + # yfinance +pyyaml==6.0.2 + # via + # langchain + # langchain-core +requests==2.32.5 + # via + # -r requirements-backend.in + # google-api-core + # langchain + # langsmith + # newsapi-python + # requests-toolbelt + # yfinance +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +six==1.17.0 + # via python-dateutil +sniffio==1.3.1 + # via anyio +soupsieve==2.8 + # via beautifulsoup4 +sqlalchemy==2.0.43 + # via langchain +starlette==0.47.3 + # via fastapi +tenacity==9.1.2 + # via langchain-core +tomli==2.2.1 + # via pytest +typing-extensions==4.15.0 + # via + # anyio + # beautifulsoup4 + # exceptiongroup + # fastapi + # langchain-core + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typing-inspection + # uvicorn +typing-inspection==0.4.1 + # via pydantic +tzdata==2025.2 + # via pandas +urllib3==2.5.0 + # via requests +uvicorn==0.35.0 + # via -r requirements-backend.in +websockets==15.0.1 + # via yfinance +xxhash==3.5.0 + # via langgraph +yfinance==0.2.65 + # via -r requirements-backend.in +zstandard==0.24.0 + # via langsmith + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements-frontend.in b/requirements-frontend.in new file mode 100644 index 0000000..1ffdfbf --- /dev/null +++ b/requirements-frontend.in @@ -0,0 +1,6 @@ +streamlit +requests +plotly +pandas +numpy +yfinance diff --git a/requirements-frontend.txt b/requirements-frontend.txt new file mode 100644 index 0000000..8a9d5a7 --- /dev/null +++ b/requirements-frontend.txt @@ -0,0 +1,136 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements-frontend.txt requirements-frontend.in +# +altair==5.5.0 + # via streamlit +attrs==25.3.0 + # via + # jsonschema + # referencing +beautifulsoup4==4.13.5 + # via yfinance +blinker==1.9.0 + # via streamlit +cachetools==6.2.0 + # via streamlit +certifi==2025.8.3 + # via + # curl-cffi + # requests +cffi==1.17.1 + # via curl-cffi +charset-normalizer==3.4.3 + # via requests +click==8.2.1 + # via streamlit +curl-cffi==0.13.0 + # via yfinance +frozendict==2.4.6 + # via yfinance +gitdb==4.0.12 + # via gitpython +gitpython==3.1.45 + # via streamlit +idna==3.10 + # via requests +jinja2==3.1.6 + # via + # altair + # pydeck +jsonschema==4.25.1 + # via altair +jsonschema-specifications==2025.4.1 + # via jsonschema +markupsafe==3.0.2 + # via jinja2 +multitasking==0.0.12 + # via yfinance +narwhals==2.2.0 + # via + # altair + # plotly +numpy==2.2.6 + # via + # -r requirements-frontend.in + # pandas + # pydeck + # streamlit + # yfinance +packaging==25.0 + # via + # altair + # plotly + # streamlit +pandas==2.3.2 + # via + # -r requirements-frontend.in + # streamlit + # yfinance +peewee==3.18.2 + # via yfinance +pillow==11.3.0 + # via streamlit +platformdirs==4.4.0 + # via yfinance +plotly==6.3.0 + # via -r requirements-frontend.in +protobuf==6.32.0 + # via + # streamlit + # yfinance +pyarrow==21.0.0 + # via streamlit +pycparser==2.22 + # via cffi +pydeck==0.9.1 + # via streamlit +python-dateutil==2.9.0.post0 + # via pandas +pytz==2025.2 + # via + # pandas + # yfinance +referencing==0.36.2 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.5 + # via + # -r requirements-frontend.in + # streamlit + # yfinance +rpds-py==0.27.1 + # via + # jsonschema + # referencing +six==1.17.0 + # via python-dateutil +smmap==5.0.2 + # via gitdb +soupsieve==2.8 + # via beautifulsoup4 +streamlit==1.49.0 + # via -r requirements-frontend.in +tenacity==9.1.2 + # via streamlit +toml==0.10.2 + # via streamlit +tornado==6.5.2 + # via streamlit +typing-extensions==4.15.0 + # via + # altair + # beautifulsoup4 + # referencing + # streamlit +tzdata==2025.2 + # via pandas +urllib3==2.5.0 + # via requests +websockets==15.0.1 + # via yfinance +yfinance==0.2.65 + # via -r requirements-frontend.in