Fetcher

1 files changed, 100 insertions, 0 deletions

diff --git a/includes/fetcher/index.js b/includes/fetcher/index.js
new file mode 100644
index 0000000..0f3dcac
--- /dev/null
+++ b/includes/fetcher/index.js
@@ -0,0 +1,100 @@
+(async () => {
+    const fs = require('fs');
+    const cp = require('child_process');
+    const axios = require('axios');
+    const crypto = require('crypto')
+
+    console.log("Fetching projects... GitLab");
+    let gitlabProjects = [];
+    let smallestId = -1;
+
+    while (smallestId !== 0) {
+        let gitlabProjectsRaw;
+
+        if (smallestId > 0) {
+            gitlabProjectsRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/projects?order_by=id&archived=false&simple=true&id_before=${smallestId}`)).data;
+        } else {
+            gitlabProjectsRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/projects?order_by=id&archived=false&simple=true`)).data;
+        }
+        for (let project of gitlabProjectsRaw) {
+            gitlabProjects.push({
+                gitlab_id: project.id,
+                youtrack_id: null,
+                name: project.name,
+                description: project.description,
+                issues: null,
+                vcs: project.http_url_to_repo,
+                web: project.web_url,
+                showcase: project.topics.includes("Showcase")
+            })
+            smallestId = project.id;
+        }
+        if (gitlabProjectsRaw.length === 0) {
+            smallestId = 0;
+        }
+    }
+
+    console.log("Fetching projects... YouTrack");
+    const youtrackProjectsRaw = (await axios.get(`https://youtrack.minteck.org/api/admin/projects?fields=id,name,shortName,description`)).data;
+    let youtrackProjects = [];
+
+    for (let project of youtrackProjectsRaw) {
+        youtrackProjects.push({
+            gitlab_id: null,
+            youtrack_id: project.id,
+            name: project.name,
+            description: project.description,
+            issues: project.shortName,
+            vcs: null,
+            web: null,
+            showcase: false
+        })
+    }
+
+    console.log("Merging data...")
+    let projects = {};
+    let projectsPlusYoutrack = {};
+    for (let project of youtrackProjects) {
+        nameCompareYoutrack = project.name.toLowerCase().replace(/[^a-z]+/gm, "");
+        descCompareYoutrack = project.description.toLowerCase().replace(/[^a-z]+/gm, "");
+
+        for (let gprj of gitlabProjects) {
+            nameCompareGitlab = gprj.name.toLowerCase().replace(/[^a-z]+/gm, "");
+            descCompareGitlab = gprj.description.toLowerCase().replace(/[^a-z]+/gm, "");
+
+            if (nameCompareGitlab === nameCompareYoutrack || descCompareGitlab === descCompareYoutrack) {
+                gprj.youtrack_id = project.youtrack_id;
+            }
+
+            if (gprj.youtrack_id === null) {
+                id = crypto.createHash('sha1').update(gprj.gitlab_id.toString() + "null").digest('hex');
+            } else {
+                id = crypto.createHash('sha1').update(gprj.gitlab_id.toString() + gprj.youtrack_id.toString()).digest('hex');
+                projectsPlusYoutrack[id] = gprj;
+            }
+            projects[id] = gprj;
+        }
+    }
+
+    const knownYoutrackIds = Object.keys(projectsPlusYoutrack).map((i) => { return projectsPlusYoutrack[i].youtrack_id; });
+    let unusedYoutrackProjects = [];
+    for (let project of youtrackProjectsRaw) {
+        if (!knownYoutrackIds.includes(project.id)) {
+            project.name_compare = project.name.toLowerCase().replace(/[^a-z]+/gm, "");
+            project.description_compare = project.description.toLowerCase().replace(/[^a-z]+/gm, "");
+            unusedYoutrackProjects.push(project);
+        }
+    }
+
+    let unusedBase = [];
+    for (let project of gitlabProjects) {
+        project.name_compare = project.name.toLowerCase().replace(/[^a-z]+/gm, "");
+        project.description_compare = project.description.toLowerCase().replace(/[^a-z]+/gm, "");
+        unusedBase.push(project);
+    }
+
+    fs.writeFileSync("projects.json", JSON.stringify(projects, false, 4));
+    fs.writeFileSync("unused-live.json", JSON.stringify(unusedYoutrackProjects, false, 4));
+    fs.writeFileSync("unused-base.json", JSON.stringify(unusedBase, false, 4));
+    console.log("Done merging, found " + Object.keys(projects).length + " projects (" + Object.keys(projectsPlusYoutrack).length + " on YouTrack, " + unusedYoutrackProjects.length + " unused)");
+})()
\ No newline at end of file