Merge pull request #9 from ahmed-n-abdeltwab/feature/make-modular

fix: permission issues and model training errors

Author: ahmed-n-abdeltwab

.github/workflows/train_and_release.yml

@@ -39,13 +39,19 @@ jobs:
           docker build -t $DOCKER_IMAGE .
           docker images
 
-      - name: Run training pipeline
+      - name: Prepare release directory
         run: |
           mkdir -p ./$RELEASE_DIR/latest
+          chmod -R 777 ./$RELEASE_DIR
+
+      - name: Run training pipeline
+        run: |
           docker run --rm \
             -v $(pwd)/data:/app/data \
             -v $(pwd)/models:/app/models \
             -v $(pwd)/$RELEASE_DIR:/app/release \
+            -e PYTHONPATH=/app \
+            -u 1001 \
             $DOCKER_IMAGE
 
       - name: Verify artifacts
@@ -57,10 +63,6 @@ jobs:
             docker logs $(docker ps -lq) || true
             exit 1
           fi
-          if [ ! -f ./$RELEASE_DIR/latest/metadata.json ]; then
-            echo "Error: metadata.json not found!"
-            exit 1
-          fi
 
       - name: Create release package
         run: |

Dockerfile

@@ -26,11 +26,11 @@ ENV PYTHONPATH=/app
 RUN mkdir -p /app && \
     mkdir -p /app/data && \
     mkdir -p /app/models && \
-    mkdir -p /app/release && \
+    mkdir -p /app/release/latest && \
     chmod -R 777 /app
 
-# Create non-root user
-RUN useradd -m appuser && \
+# Create non-root user with specific UID
+RUN useradd -u 1001 -m appuser && \
     chown -R appuser:appuser /app
 
 WORKDIR /app
@@ -41,8 +41,10 @@ COPY --from=builder --chown=appuser:appuser /opt/venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
 # Copy application files
-COPY --chown=appuser:appuser src/ ./src/
-COPY --chown=appuser:appuser config/ ./config/
+COPY --chown=appuser:appuser . .
+
+# Install package in development mode
+RUN pip install -e .
 
 # Health check
 HEALTHCHECK --interval=30s --timeout=10s \

config/components/model_trainer.yaml

@@ -9,7 +9,7 @@ params:
   hyperparams:
     random_forest:
       n_estimators: [100, 200]
-      max_depth: [10, 20, 30]  # Remove None from options
+      max_depth: [10, 20]  # Removed None
       min_samples_split: [2, 5]
     svm:
       C: [0.1, 1, 10]

src/components/artifact_exporter.py

@@ -10,9 +10,11 @@
 class ArtifactExporter:
     def __init__(self, config: Dict[str, Any]):
         self.config = config
-        # Ensure absolute path and proper directory structure
         self.config["output_dir"] = os.path.abspath(config.get("output_dir", "release"))
+
+        # Ensure directory exists with write permissions
         os.makedirs(os.path.join(self.config["output_dir"], "latest"), exist_ok=True)
+        os.chmod(os.path.join(self.config["output_dir"], "latest"), 0o777)
 
     def export_training_artifacts(
         self, training_results: Dict[str, Any]
@@ -24,6 +26,7 @@ def export_training_artifacts(
             # Clear existing contents
             shutil.rmtree(release_dir, ignore_errors=True)
             os.makedirs(release_dir, exist_ok=True)
+            os.chmod(release_dir, 0o777)
 
             exported_files = {}
             artifacts = training_results["artifacts"]
@@ -42,6 +45,7 @@ def export_training_artifacts(
                 if src and os.path.exists(src):
                     dest = os.path.join(release_dir, filename)
                     shutil.copy2(src, dest)
+                    os.chmod(dest, 0o666)  # Ensure writable
                     exported_files[filename.split(".")[0]] = dest
                 elif filename == "metrics.json":
                     # Create default metrics if missing
@@ -71,8 +75,10 @@ def _create_default_metrics(
                 "warning": "Metrics not properly saved during training",
             },
         )
-        with open(os.path.join(release_dir, "metrics.json"), "w") as f:
+        metrics_path = os.path.join(release_dir, "metrics.json")
+        with open(metrics_path, "w") as f:
             json.dump(default_metrics, f, indent=2)
+        os.chmod(metrics_path, 0o666)
 
     def _create_feature_structure(
         self, release_dir: str, training_results: Dict[str, Any]
@@ -82,14 +88,18 @@ def _create_feature_structure(
             "required_features": len(training_results.get("selected_features", [])),
             "version": datetime.now().strftime("%Y%m%d_%H%M%S"),
         }
-        with open(os.path.join(release_dir, "feature_structure.json"), "w") as f:
+        features_path = os.path.join(release_dir, "feature_structure.json")
+        with open(features_path, "w") as f:
             json.dump(feature_structure, f, indent=2)
+        os.chmod(features_path, 0o666)
 
     def _create_package_info(self, release_dir: str, files: Dict[str, str]):
         package_info = {
             "created_at": datetime.now().isoformat(),
             "contents": {k: os.path.basename(v) for k, v in files.items()},
             "notes": "Automatically generated by spyware-detector-training pipeline",
         }
-        with open(os.path.join(release_dir, "package_info.json"), "w") as f:
+        info_path = os.path.join(release_dir, "package_info.json")
+        with open(info_path, "w") as f:
             json.dump(package_info, f, indent=2)
+        os.chmod(info_path, 0o666)

src/main.py

@@ -25,10 +25,14 @@ def main():
     try:
         logger.info("🚀 Starting Spyware Detector Training Pipeline")
 
-        # Ensure release directory exists
+        # Ensure release directory exists with proper permissions
         release_dir = os.path.abspath("release/latest")
         os.makedirs(release_dir, exist_ok=True)
+        os.chmod(release_dir, 0o777)
         logger.info(f"Release directory: {release_dir}")
+        logger.info(
+            f"Directory permissions: {oct(os.stat(release_dir).st_mode & 0o777)}"
+        )
 
         # Initialize and run pipeline
         pipeline = TrainingPipeline("config/pipeline.yaml")
@@ -42,14 +46,18 @@ def main():
         shutil.make_archive("model_release", "zip", release_dir)
         logger.info("📦 Created release package: model_release.zip")
 
-        # Log exported files
-        logger.info("💾 Artifacts generated:")
+        # Verify all artifacts were created
+        missing_files = []
         for name, path in results["exported_files"].items():
             if path and os.path.exists(path):
                 logger.info(f"  - {name}: {path}")
             else:
+                missing_files.append(name)
                 logger.warning(f"  - {name}: FILE MISSING")
 
+        if missing_files:
+            raise RuntimeError(f"Missing exported files: {', '.join(missing_files)}")
+
     except Exception as e:
         logger.error(f"❌ Pipeline failed: {str(e)}", exc_info=True)
         sys.exit(1)