Merge pull request #5 from ahmed-n-abdeltwab/feature/make-modular

Refactor: Transform project into a modular and dynamic structure

Author: ahmed-n-abdeltwab

.dockerignore

@@ -1,11 +1,14 @@
-__pycache__/
-*.pyc
-*.pyo
-*.pyd
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
 venv/
 .env
-data/raw/
-data/processed/
-models/saved/
-__init__.py
 .DS_Store
+*.log
+*.sqlite3
+*.zip
+*.tar.gz
+.pytest_cache/
+.idea/
+.vscode/

.github/workflows/train_and_release.yml

@@ -2,46 +2,80 @@ name: Train and Release Model
 
 on:
   push:
-    branches:
-      - main  # Trigger when pushing to main
-  workflow_dispatch:  # Allow manual trigger
+    branches: [ main ]
+  workflow_dispatch:
+
+env:
+  DOCKER_IMAGE: spyware-detector
+  RELEASE_DIR: release
 
 jobs:
   train-and-release:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      packages: write
+      actions: read
+
     steps:
-      - name: Checkout Repository
+      - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Set up Python
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 
-      - name: Install Dependencies
+      - name: Install dependencies
         run: |
+          python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      - name: Train the Model
+      - name: Build Docker image
+        run: docker build -t $DOCKER_IMAGE .
+
+      - name: Run training pipeline
         run: |
-          python main.py  # Modify if needed to run training
+          mkdir -p ./$RELEASE_DIR
+          docker run --rm \
+            -v $(pwd)/data:/app/data \
+            -v $(pwd)/models:/app/models \
+            -v $(pwd)/$RELEASE_DIR:/app/$RELEASE_DIR \
+            $DOCKER_IMAGE
 
-      - name: Archive Model
+      - name: Verify artifacts
         run: |
-          mkdir -p release
-          cp models/saved/*/model.pkl release/model.pkl
-          cp models/saved/*/metadata.json release/metadata.json
-          cp models/saved/*/metrics.json release/metrics.json
-          tar -czvf trained_model.tar.gz -C release .
+          echo "Generated artifacts:"
+          ls -lR ./$RELEASE_DIR
+          [ -f ./$RELEASE_DIR/latest/model.pkl ] || exit 1
+          [ -f ./$RELEASE_DIR/latest/metadata.json ] || exit 1
+
+      - name: Create release package
+        run: |
+          TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+          RELEASE_NAME="model_$TIMESTAMP"
+          mkdir -p ./$RELEASE_DIR/$RELEASE_NAME
+          cp -r ./$RELEASE_DIR/latest/* ./$RELEASE_DIR/$RELEASE_NAME/
+          tar -czvf ./$RELEASE_DIR/$RELEASE_NAME.tar.gz -C ./$RELEASE_DIR/$RELEASE_NAME .
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: model-release
+          path: ./$RELEASE_DIR/*.tar.gz
 
       - name: Create GitHub Release
+        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v1
         with:
-          tag_name: latest-model-${{ github.run_number }}
-          name: Latest Trained Model
-          body: "This model was trained automatically on commit ${{ github.sha }}"
-          files: trained_model.tar.gz
+          name: "Model Release $(date +'%Y-%m-%d')"
+          body: |
+            Automated model training results:
+            - Model: $(jq -r '.model_type' ./$RELEASE_DIR/latest/metadata.json)
+            - Timestamp: $(jq -r '.timestamp' ./$RELEASE_DIR/latest/metadata.json)
+            - Metrics:
+              Accuracy: $(jq -r '.metrics.accuracy' ./$RELEASE_DIR/latest/metrics.json)
+              F1 Score: $(jq -r '.metrics.f1' ./$RELEASE_DIR/latest/metrics.json)
+          files: ./$RELEASE_DIR/*.tar.gz
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT }}

.gitignore

@@ -1,12 +1,53 @@
+# Byte-compiled / optimized / DLL files
 __pycache__/
-*.pyc
-*.pyo
-*.pyd
+*.py[cod]
+*$py.class
+
+# Virtual environment
 venv/
-.env
+.env/
+.venv/
+
+# IDE specific files
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Data files
+data/raw/
 data/processed/
+*.csv
+*.pkl
+*.pickle
+
+# Model files
 models/saved/
-.DS_Store
-__init__.py
+*.pth
+*.h5
+*.joblib
+
+# Log files
 *.log
-*.sqlite3
+logs/
+
+# System files
+.DS_Store
+Thumbs.db
+
+# Release packages
+release/
+*.zip
+*.tar.gz
+
+# Test coverage
+.coverage
+htmlcov/
+.pytest_cache/
+
+# Jupyter notebooks
+*.ipynb
+.ipynb_checkpoints/
+
+# Documentation
+docs/_build/

Dockerfile

@@ -1,15 +1,49 @@
-FROM python:3.9-slim
+# ========== BUILD STAGE ==========
+FROM python:3.9 AS builder
 
-# Set the working directory
 WORKDIR /app
 
-# Copy requirement files and install dependencies
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends gcc python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install Python dependencies
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy the application code
-COPY . .
 
-# Set the entry point
-CMD ["python", "main.py"]
+# ========== RUNTIME STAGE ==========
+FROM python:3.9-slim
+
+ENV PYTHONPATH=/app
+
+# Create non-root user
+RUN useradd -m appuser && \
+    mkdir -p /app && \
+    chown appuser:appuser /app
+
+WORKDIR /app
+USER appuser
+
+# Copy virtual environment
+COPY --from=builder /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy application files
+COPY --chown=appuser:appuser src/ ./src/
+COPY --chown=appuser:appuser config/ ./config/
+
+# Create data directories
+RUN mkdir -p data/raw data/processed models/saved release
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s \
+  CMD python -c "import sys; sys.exit(0)"
 
+# Entry point
+CMD ["python", "src/main.py"]