diff --git a/controllers/partial/saveData.js b/controllers/partial/saveData.js index 7c56f99..64eb65f 100644 --- a/controllers/partial/saveData.js +++ b/controllers/partial/saveData.js @@ -6,9 +6,21 @@ const saveDataToDatabase = async (_kwarq) => { const { data, id, defaultDB } = _kwarq; if (!data) return console.log( - "Skipping saving record. Content is not relevant to Accelerator lab. URL: " - + data?.url + "Skipping saving record. Content is not relevant to Accelerator lab. URL: " + + data?.url ); + + if ( + (data.parsed_date && new Date(data.parsed_date) < new Date("2019-01-01")) || + (data.postedDate && new Date(data.postedDate) < new Date("2019-01-01")) + ) { + console.log( + "Skipping saving record. Content is not relevant to Accelerator Lab. Date is before Acclabs. URL: " + + data?.url + ); + return; + } + try { let db = defaultDB ?? DB.blog; let embedding_id; @@ -41,7 +53,7 @@ const saveDataToDatabase = async (_kwarq) => { data.postedDateStr, data.iso3, data.parsed_date, - data.relevance || 2 + data.relevance || 2, ] ); @@ -75,7 +87,7 @@ const saveDataToDatabase = async (_kwarq) => { ) ); - embedding_id = record?.id + embedding_id = record?.id; } else { // Update existing record batch.push( @@ -126,25 +138,25 @@ const saveDataToDatabase = async (_kwarq) => { [id, data.raw_html] ) ); - - embedding_id = id + + embedding_id = id; } return t.batch(batch).catch((err) => console.log(err)); }); console.log("Saving record successful."); - // Embed document using the NLP API - if(embedding_id){ + // Embed document using the NLP API + if (embedding_id) { embedDocument(embedding_id) - .then(() => { - console.log('Document embedded successfully.'); - }) - .catch((error) => { - console.error('Error executing function:', error); - //Todo: what happens when a document embedding fails? - }); - } + .then(() => { + console.log("Document embedded successfully."); + }) + .catch((error) => { + console.error("Error executing function:", error); + //Todo: what happens when a document embedding fails? + }); + } } catch (error) { console.error("Error saving data to database:", error); }