Merge pull request #11 from ahmed-n-abdeltwab/feature/make-modular

fix: artifact packaging and upload in CI workflow

Author: ahmed-n-abdeltwab

.github/workflows/train_and_release.yml

@@ -56,25 +56,55 @@ jobs:
 
       - name: Verify artifacts
         run: |
-          echo "Generated artifacts:"
-          ls -lR ./$RELEASE_DIR
-          if [ ! -f ./$RELEASE_DIR/latest/model.pkl ]; then
-            echo "Error: model.pkl not found!"
+          echo "Generated artifacts in release/latest:"
+          ls -lR ./$RELEASE_DIR/latest
+          
+          REQUIRED_FILES=(
+            "model.pkl"
+            "metadata.json"
+            "metrics.json"
+            "feature_structure.json"
+          )
+          
+          missing_files=0
+          for file in "${REQUIRED_FILES[@]}"; do
+            if [ ! -f "./$RELEASE_DIR/latest/$file" ]; then
+              echo "Error: Required file $file not found!"
+              missing_files=$((missing_files+1))
+            fi
+          done
+          
+          if [ $missing_files -gt 0 ]; then
+            echo "Error: Missing $missing_files required files!"
             docker logs $(docker ps -lq) || true
             exit 1
           fi
 
       - name: Create release package
         run: |
+          echo "Contents of release directory before packaging:"
+          ls -lR ./$RELEASE_DIR
+          
           TIMESTAMP=$(date +%Y%m%d_%H%M%S)
           RELEASE_NAME="model_$TIMESTAMP"
-          tar -czvf ./$RELEASE_DIR/$RELEASE_NAME.tar.gz -C ./$RELEASE_DIR/latest .
+          
+          mkdir -p ./$RELEASE_DIR/temp_package
+          cp -r ./$RELEASE_DIR/latest/* ./$RELEASE_DIR/temp_package/
+          
+          tar -czvf ./$RELEASE_DIR/$RELEASE_NAME.tar.gz -C ./$RELEASE_DIR/temp_package .
+          
+          rm -rf ./$RELEASE_DIR/temp_package
+          
+          echo "Created package:"
+          ls -l ./$RELEASE_DIR/*.tar.gz
 
       - name: Upload artifact
+        if: success()
         uses: actions/upload-artifact@v4
         with:
           name: model-release
-          path: ./$RELEASE_DIR/*.tar.gz
+          path: ${{ env.RELEASE_DIR }}/*.tar.gz
+          if-no-files-found: error
 
       - name: Create GitHub Release
         if: startsWith(github.ref, 'refs/tags/')
@@ -85,6 +115,9 @@ jobs:
             Automated model training results:
             - Model: $(jq -r '.model_type' ./$RELEASE_DIR/latest/metadata.json)
             - Timestamp: $(jq -r '.timestamp' ./$RELEASE_DIR/latest/metadata.json)
+            - Metrics:
+              Accuracy: $(jq -r '.metrics.accuracy' ./$RELEASE_DIR/latest/metrics.json)
+              F1 Score: $(jq -r '.metrics.f1' ./$RELEASE_DIR/latest/metrics.json)
           files: ./$RELEASE_DIR/*.tar.gz
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

src/components/artifact_exporter.py

@@ -11,24 +11,28 @@ class ArtifactExporter:
     def __init__(self, config: Dict[str, Any]):
         self.config = config
         self.config["output_dir"] = os.path.abspath(config.get("output_dir", "release"))
+        self._ensure_output_directory()
 
-        # Ensure directory exists with write permissions
-        os.makedirs(os.path.join(self.config["output_dir"], "latest"), exist_ok=True)
-        os.chmod(os.path.join(self.config["output_dir"], "latest"), 0o777)
+    def _ensure_output_directory(self):
+        """Ensure output directory exists with proper permissions"""
+        release_dir = os.path.join(self.config["output_dir"], "latest")
+        os.makedirs(release_dir, exist_ok=True)
+        os.chmod(release_dir, 0o777)
+        return release_dir
 
     def export_training_artifacts(
         self, training_results: Dict[str, Any]
     ) -> Dict[str, str]:
         """Export all artifacts to release/latest directory"""
-        try:
-            release_dir = os.path.join(self.config["output_dir"], "latest")
+        release_dir = self._ensure_output_directory()
+        exported_files = {}
 
+        try:
             # Clear existing contents
             shutil.rmtree(release_dir, ignore_errors=True)
             os.makedirs(release_dir, exist_ok=True)
             os.chmod(release_dir, 0o777)
 
-            exported_files = {}
             artifacts = training_results["artifacts"]
 
             # Export model files
@@ -42,29 +46,35 @@ def export_training_artifacts(
             }
 
             for filename, src in model_files.items():
+                dest = os.path.join(release_dir, filename)
                 if src and os.path.exists(src):
-                    dest = os.path.join(release_dir, filename)
                     shutil.copy2(src, dest)
-                    os.chmod(dest, 0o666)  # Ensure writable
+                    os.chmod(dest, 0o644)
                     exported_files[filename.split(".")[0]] = dest
                 elif filename == "metrics.json":
-                    # Create default metrics if missing
                     self._create_default_metrics(release_dir, training_results)
                     exported_files["metrics"] = os.path.join(release_dir, filename)
 
             # Create supporting files
             self._create_feature_structure(release_dir, training_results)
             self._create_package_info(release_dir, exported_files)
 
-            return {k: os.path.abspath(v) for k, v in exported_files.items()}
+            # Verify all files were created
+            for path in exported_files.values():
+                if not os.path.exists(path):
+                    raise FileNotFoundError(f"Failed to create artifact at {path}")
+
+            return exported_files
 
         except Exception as e:
-            print(f"Artifact export failed: {str(e)}")
-            raise
+            # Clean up partial exports
+            shutil.rmtree(release_dir, ignore_errors=True)
+            raise RuntimeError(f"Failed to export artifacts: {str(e)}")
 
     def _create_default_metrics(
         self, release_dir: str, training_results: Dict[str, Any]
     ):
+        metrics_path = os.path.join(release_dir, "metrics.json")
         default_metrics = training_results.get(
             "metrics",
             {
@@ -75,31 +85,30 @@ def _create_default_metrics(
                 "warning": "Metrics not properly saved during training",
             },
         )
-        metrics_path = os.path.join(release_dir, "metrics.json")
         with open(metrics_path, "w") as f:
             json.dump(default_metrics, f, indent=2)
-        os.chmod(metrics_path, 0o666)
+        os.chmod(metrics_path, 0o644)
 
     def _create_feature_structure(
         self, release_dir: str, training_results: Dict[str, Any]
     ):
+        features_path = os.path.join(release_dir, "feature_structure.json")
         feature_structure = {
             "feature_names": training_results.get("selected_features", []),
             "required_features": len(training_results.get("selected_features", [])),
             "version": datetime.now().strftime("%Y%m%d_%H%M%S"),
         }
-        features_path = os.path.join(release_dir, "feature_structure.json")
         with open(features_path, "w") as f:
             json.dump(feature_structure, f, indent=2)
-        os.chmod(features_path, 0o666)
+        os.chmod(features_path, 0o644)
 
     def _create_package_info(self, release_dir: str, files: Dict[str, str]):
+        info_path = os.path.join(release_dir, "package_info.json")
         package_info = {
             "created_at": datetime.now().isoformat(),
             "contents": {k: os.path.basename(v) for k, v in files.items()},
             "notes": "Automatically generated by spyware-detector-training pipeline",
         }
-        info_path = os.path.join(release_dir, "package_info.json")
         with open(info_path, "w") as f:
             json.dump(package_info, f, indent=2)
-        os.chmod(info_path, 0o666)
+        os.chmod(info_path, 0o644)

src/main.py

@@ -38,25 +38,30 @@ def main():
         pipeline = TrainingPipeline("config/pipeline.yaml")
         results = pipeline.run()
 
-        # Log results
-        logger.info("✅ Training completed successfully")
-        logger.info(f"📊 Model Metrics:\n{json.dumps(results['metrics'], indent=2)}")
-
-        # Create compressed release package
-        shutil.make_archive("model_release", "zip", release_dir)
-        logger.info("📦 Created release package: model_release.zip")
-
-        # Verify all artifacts were created
+        # Verify exported files exist
+        logger.info("Verifying exported files:")
         missing_files = []
         for name, path in results["exported_files"].items():
             if path and os.path.exists(path):
                 logger.info(f"  - {name}: {path}")
+                os.chmod(path, 0o644)
             else:
+                logger.error(f"  - {name}: FILE MISSING")
                 missing_files.append(name)
-                logger.warning(f"  - {name}: FILE MISSING")
 
         if missing_files:
-            raise RuntimeError(f"Missing exported files: {', '.join(missing_files)}")
+            raise FileNotFoundError(
+                f"Missing exported files: {', '.join(missing_files)}"
+            )
+
+        # Log results
+        logger.info("✅ Training completed successfully")
+        logger.info(f"📊 Model Metrics:\n{json.dumps(results['metrics'], indent=2)}")
+
+        # Create compressed release package
+        archive_path = os.path.join(release_dir, "model_release.zip")
+        shutil.make_archive(archive_path[:-4], "zip", release_dir)
+        logger.info(f"📦 Created release package: {archive_path}")
 
     except Exception as e:
         logger.error(f"❌ Pipeline failed: {str(e)}", exc_info=True)