aboutsummaryrefslogtreecommitdiff
path: root/includes/fetcher/index.js
diff options
context:
space:
mode:
Diffstat (limited to 'includes/fetcher/index.js')
-rw-r--r--includes/fetcher/index.js108
1 files changed, 103 insertions, 5 deletions
diff --git a/includes/fetcher/index.js b/includes/fetcher/index.js
index 0b36111..edcbcf3 100644
--- a/includes/fetcher/index.js
+++ b/includes/fetcher/index.js
@@ -12,14 +12,22 @@
let gitlabProjectsRaw;
if (smallestId > 0) {
- gitlabProjectsRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/users/minteck/projects?order_by=id&archived=false&simple=true&id_before=${smallestId}`)).data;
+ gitlabProjectsRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/users/minteck/projects?order_by=id&archived=false&id_before=${smallestId}`)).data;
} else {
- gitlabProjectsRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/users/minteck/projects?order_by=id&archived=false&simple=true`)).data;
+ gitlabProjectsRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/users/minteck/projects?order_by=id&archived=false`)).data;
}
for (let project of gitlabProjectsRaw) {
+ let readme;
+ try {
+ readme = Buffer.from((await axios.get("https://gitlab.minteck.org/api/v4/projects/" + project.id + "/repository/blobs/" + (await axios.get("https://gitlab.minteck.org/api/v4/projects/" + project.id + "/repository/tree")).data.filter(i => i.type === "blob" && (i.name.toLowerCase() === "readme.md" || i.name.toLowerCase() === "readme"))[0].id + "/raw")).data).toString("base64")
+ } catch (e) {
+ readme = "";
+ }
gitlabProjects.push({
+ archive: false,
gitlab_id: project.id,
youtrack_id: null,
+ path: project.path_with_namespace,
name: project.name,
description: project.description,
issues: null,
@@ -27,15 +35,62 @@
web: project.web_url,
icon: project.avatar_url,
showcase: project.topics.includes("Showcase"),
- date: project.last_activity_at
+ date: project.last_activity_at,
+ event: (await axios.get(`https://gitlab.minteck.org/api/v4/projects/${project.id}/events`)).data[0],
+ tags: project.topics,
+ readme: readme
})
smallestId = project.id;
+
}
if (gitlabProjectsRaw.length === 0) {
smallestId = 0;
}
}
+ console.log("Fetching projects... GitLab archives");
+ let gitlabArchiveProjects = [];
+ smallestId = -1;
+
+ while (smallestId !== 0) {
+ let gitlabProjectsArchiveRaw;
+
+ if (smallestId > 0) {
+ gitlabProjectsArchiveRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/users/minteck/projects?order_by=id&archived=true&id_before=${smallestId}`)).data;
+ } else {
+ gitlabProjectsArchiveRaw = (await axios.get(`https://gitlab.minteck.org/api/v4/users/minteck/projects?order_by=id&archived=true`)).data;
+ }
+ for (let project of gitlabProjectsArchiveRaw) {
+ let readme;
+ try {
+ readme = Buffer.from((await axios.get("https://gitlab.minteck.org/api/v4/projects/" + project.id + "/repository/blobs/" + (await axios.get("https://gitlab.minteck.org/api/v4/projects/" + project.id + "/repository/tree")).data.filter(i => i.type === "blob" && (i.name.toLowerCase() === "readme.md" || i.name.toLowerCase() === "readme"))[0].id + "/raw")).data).toString("base64")
+ } catch (e) {
+ readme = "";
+ }
+ gitlabProjects.push({
+ archive: true,
+ gitlab_id: project.id,
+ youtrack_id: null,
+ path: project.path_with_namespace,
+ name: project.name,
+ description: project.description,
+ issues: null,
+ vcs: project.http_url_to_repo,
+ web: project.web_url,
+ icon: project.avatar_url,
+ showcase: project.topics.includes("Showcase"),
+ date: project.last_activity_at,
+ event: (await axios.get(`https://gitlab.minteck.org/api/v4/projects/${project.id}/events`)).data[0],
+ tags: project.topics,
+ readme: readme
+ })
+ smallestId = project.id;
+ }
+ if (gitlabProjectsArchiveRaw.length === 0) {
+ smallestId = 0;
+ }
+ }
+
console.log("Fetching projects... YouTrack");
let youtrackProjects = [];
let unusedBase = [];
@@ -47,6 +102,7 @@
const youtrackProjectsRaw = (await axios.get(`https://youtrack.minteck.org/api/admin/projects?fields=id,name,shortName,description`)).data;
for (let project of youtrackProjectsRaw) {
youtrackProjects.push({
+ archive: false,
gitlab_id: null,
youtrack_id: project.id,
name: project.name,
@@ -56,7 +112,10 @@
web: null,
icon: null,
showcase: false,
- date: null
+ date: null,
+ event: null,
+ tags: [],
+ readme: ""
})
}
@@ -71,7 +130,25 @@
if (nameCompareGitlab === nameCompareYoutrack || descCompareGitlab === descCompareYoutrack) {
gprj.youtrack_id = project.youtrack_id;
- gprj.issues = "https://youtrack.minteck.org/issues/" + project.youtrack_id;
+ gprj.issues = project.youtrack_id;
+ }
+
+ if (gprj.youtrack_id === null) {
+ id = crypto.createHash('sha1').update(gprj.gitlab_id.toString() + "null").digest('hex');
+ } else {
+ id = crypto.createHash('sha1').update(gprj.gitlab_id.toString() + gprj.youtrack_id.toString()).digest('hex');
+ projectsPlusYoutrack[id] = gprj;
+ }
+ projects[id] = gprj;
+ }
+
+ for (let gprj of gitlabArchiveProjects) {
+ nameCompareGitlab = gprj.name.toLowerCase().replace(/[^a-z]+/gm, "");
+ descCompareGitlab = gprj.description.toLowerCase().replace(/[^a-z]+/gm, "");
+
+ if (nameCompareGitlab === nameCompareYoutrack || descCompareGitlab === descCompareYoutrack) {
+ gprj.youtrack_id = project.youtrack_id;
+ gprj.issues = project.youtrack_id;
}
if (gprj.youtrack_id === null) {
@@ -135,4 +212,25 @@
fs.writeFileSync("projects.json", JSON.stringify(projects, null, 4));
console.log("Done fetching, found " + Object.keys(projects).length + " projects");
}
+
+ if (fs.existsSync("personal")) fs.rmSync("personal", { recursive: true })
+ console.log("Cloning minteck/minteck from GitHub...");
+ cp.execSync("git clone https://github.com/minteck/minteck personal");
+ let list = [];
+ for (let project of require('./projects.json')) {
+ if (!project.archive) {
+ list.push(project.name + " | [" + project.path + "](https://gitlab.minteck.org/" + project.path + ")");
+ }
+ }
+ for (let project of require('./projects.json')) {
+ if (project.archive) {
+ list.push("*" + project.name + "* | *[" + project.path + "](https://gitlab.minteck.org/" + project.path + ")*");
+ }
+ }
+ console.log("Saving...");
+ fs.writeFileSync("./personal/README.md", "<!-- WARNING: Do not modify this file, modify README.mdt instead. This file will get overwritten whenever the project fetcher runs. -->\n" + fs.readFileSync("./personal/README.mdt").toString().replace("%GITLABFILLHERE%", list.join("\n")));
+ console.log("Publishing changes to GitHub...");
+ cp.execSync("git add -A");
+ cp.execSync("git commit -m \"Update: " + new Date().toISOString() + "\"");
+ cp.execSync("git push origin main");
})() \ No newline at end of file
© Floofi Systems, Some rights reserved. Powered by cgit.