fix: artifact generation and CI pipeline

- Standardized on 'release/latest' directory structure
- Fixed Docker volume permissions for appuser
- Added better error logging and verification
- Updated workflow to handle paths consistently
- Improved artifact export reliability

Fixes issue where release artifacts weren't being properly generated

Author: ahmed-n-abdeltwab

Diff for: .github/workflows/train_and_release.yml

@@ -35,31 +35,38 @@ jobs:
         run: sudo apt-get install -y jq
 
       - name: Build Docker image
-        run: docker build -t $DOCKER_IMAGE .
+        run: |
+          docker build -t $DOCKER_IMAGE .
+          docker images
 
       - name: Run training pipeline
         run: |
-          mkdir -p ./$RELEASE_DIR
+          mkdir -p ./$RELEASE_DIR/latest
           docker run --rm \
             -v $(pwd)/data:/app/data \
             -v $(pwd)/models:/app/models \
-            -v $(pwd)/$RELEASE_DIR:/app/$RELEASE_DIR \
+            -v $(pwd)/$RELEASE_DIR:/app/release \
             $DOCKER_IMAGE
 
       - name: Verify artifacts
         run: |
           echo "Generated artifacts:"
           ls -lR ./$RELEASE_DIR
-          [ -f ./$RELEASE_DIR/latest/model.pkl ] || exit 1
-          [ -f ./$RELEASE_DIR/latest/metadata.json ] || exit 1
+          if [ ! -f ./$RELEASE_DIR/latest/model.pkl ]; then
+            echo "Error: model.pkl not found!"
+            docker logs $(docker ps -lq) || true
+            exit 1
+          fi
+          if [ ! -f ./$RELEASE_DIR/latest/metadata.json ]; then
+            echo "Error: metadata.json not found!"
+            exit 1
+          fi
 
       - name: Create release package
         run: |
           TIMESTAMP=$(date +%Y%m%d_%H%M%S)
           RELEASE_NAME="model_$TIMESTAMP"
-          mkdir -p ./$RELEASE_DIR/$RELEASE_NAME
-          cp -r ./$RELEASE_DIR/latest/* ./$RELEASE_DIR/$RELEASE_NAME/
-          tar -czvf ./$RELEASE_DIR/$RELEASE_NAME.tar.gz -C ./$RELEASE_DIR/$RELEASE_NAME .
+          tar -czvf ./$RELEASE_DIR/$RELEASE_NAME.tar.gz -C ./$RELEASE_DIR/latest .
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4
@@ -76,9 +83,6 @@ jobs:
             Automated model training results:
             - Model: $(jq -r '.model_type' ./$RELEASE_DIR/latest/metadata.json)
             - Timestamp: $(jq -r '.timestamp' ./$RELEASE_DIR/latest/metadata.json)
-            - Metrics:
-              Accuracy: $(jq -r '.metrics.accuracy' ./$RELEASE_DIR/latest/metrics.json)
-              F1 Score: $(jq -r '.metrics.f1' ./$RELEASE_DIR/latest/metrics.json)
           files: ./$RELEASE_DIR/*.tar.gz
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Diff for: Dockerfile

@@ -1,5 +1,5 @@
 # ========== BUILD STAGE ==========
-FROM python:3.9 AS builder
+FROM python:3.9-slim AS builder
 
 WORKDIR /app
 
@@ -16,31 +16,31 @@ ENV PATH="/opt/venv/bin:$PATH"
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-
 # ========== RUNTIME STAGE ==========
 FROM python:3.9-slim
 
-ENV PYTHONPATH=/app
+# Create directory structure with proper permissions
+RUN mkdir -p /app && \
+    mkdir -p /app/data && \
+    mkdir -p /app/models && \
+    mkdir -p /app/release && \
+    chmod -R 777 /app
 
 # Create non-root user
 RUN useradd -m appuser && \
-    mkdir -p /app && \
-    chown appuser:appuser /app
+    chown -R appuser:appuser /app
 
 WORKDIR /app
 USER appuser
 
 # Copy virtual environment
-COPY --from=builder /opt/venv /opt/venv
+COPY --from=builder --chown=appuser:appuser /opt/venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
 # Copy application files
 COPY --chown=appuser:appuser src/ ./src/
 COPY --chown=appuser:appuser config/ ./config/
 
-# Create data directories
-RUN mkdir -p data/raw data/processed models/saved release
-
 # Health check
 HEALTHCHECK --interval=30s --timeout=10s \
   CMD python -c "import sys; sys.exit(0)"

Diff for: requirements.txt

@@ -2,3 +2,4 @@ pandas==1.5.3
 numpy==1.23.5
 scikit-learn==1.2.2
 PyYAML==6.0
+python-dotenv==0.19.0  # For environment variables