-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathconfig.js
161 lines (107 loc) · 5.69 KB
/
config.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
// ###########################################
// ##### Import required modules (START) #####
// ###########################################
import * as log from "deno-std-log";
// #########################################
// ##### Import required modules (END) #####
// #########################################
// ##############################################################
// ##### Information for console debugging purposes (START) #####
// ##############################################################
const filename_this = "Log(config.js): ";
console.log(filename_this + "Started reading commands from this script for processing!");
export const settings = {
data: {
pull: true,
url: "https://query.wikidata.org/sparql?format=json&query=SELECT%20DISTINCT%20%3Fitem%20%3FitemLabel%20%3Flicense%20%3FlicenseLabel%20%3Ftopic%20%3FtopicLabel%0AWHERE%20%7B%0A%20%20%3Fitem%20wdt%3AP6104%20wd%3AQ56241615%3B%0A%20%20%20%20%20%20%20%20wdt%3AP356%20%3Fdoi%20%3B%0A%20%20%20%20%20%20%20%20wdt%3AP921%20%3Ftopic%20.%0A%20%0A%20%20%7B%3Ftopic%20wdt%3AP31%20wd%3AQ41719%20.%7D%0A%20%20UNION%0A%20%20%7BVALUES%20%3Ftopic%20%7B%20wd%3AQ109467185%20wd%3AQ112148709%20wd%3AQ111525751%20wd%3AQ113019190%7D%20%7D%0A%20%20UNION%0A%20%20%7B%3Fitem%20wdt%3AP275%20%3Flicense%20.%0A%20%20%3Flicense%20wdt%3AP31*%2Fwdt%3AP279*%20wd%3AQ30939938%20.%7D%0A%20%20SERVICE%20wikibase%3Alabel%20%7B%20bd%3AserviceParam%20wikibase%3Alanguage%20%22%5BAUTO_LANGUAGE%5D%2Cen%22.%20%7D%0A%7D%0AGROUP%20BY%20%3Fitem%20%3FitemLabel%20%3Flicense%20%3FlicenseLabel%20%3Ftopic%20%3FtopicLabel",
// ##### Note: Adjustments in Wikidata can be made with:
// ##### Note: https://query.wikidata.org/#SELECT%20DISTINCT%20%3Fitem%20%3FitemLabel%20%3Flicense%20%3FlicenseLabel%20%3Ftopic%20%3FtopicLabel%0AWHERE%20%7B%0A%20%20%3Fitem%20wdt%3AP6104%20wd%3AQ56241615%3B%0A%20%20%20%20%20%20%20%20wdt%3AP356%20%3Fdoi%20%3B%0A%20%20%20%20%20%20%20%20wdt%3AP921%20%3Ftopic%20.%0A%20%0A%20%20%7B%3Ftopic%20wdt%3AP31%20wd%3AQ41719%20.%7D%0A%20%20UNION%0A%20%20%7BVALUES%20%3Ftopic%20%7B%20wd%3AQ109467185%20wd%3AQ112148709%20wd%3AQ111525751%20wd%3AQ113019190%7D%20%7D%0A%20%20UNION%0A%20%20%7B%3Fitem%20wdt%3AP275%20%3Flicense%20.%0A%20%20%3Flicense%20wdt%3AP31*%2Fwdt%3AP279*%20wd%3AQ30939938%20.%7D%0A%20%20SERVICE%20wikibase%3Alabel%20%7B%20bd%3AserviceParam%20wikibase%3Alanguage%20%22%5BAUTO_LANGUAGE%5D%2Cen%22.%20%7D%0A%7D%0AGROUP%20BY%20%3Fitem%20%3FitemLabel%20%3Flicense%20%3FlicenseLabel%20%3Ftopic%20%3FtopicLabel
items: null,
read: true,
// ##### Note: if filename to save entries.json is not provided inside (deno.jsonc), then
// the code will consider the name given below.
file: "./corpus/entries_log2.json", // ##### Note: filename to save entries, if not provided inside (deno.jsonc)
reduce: true, // ##### Note: when reading from URL, it will reduce this file to single URL requests. Then processing time is much reduced.
},
processing: {
initialOffset: 0, // ##### Note: Initial offset to start extraction process
processingDelay: 1000, // ##### Note: resting time between batches (in milliseconds)
batchSize: 1, // ##### Note: quantity of consecutive wikidataItem-URL-extraction without resting
},
};
export const logging = {
// ##### Note: from: https://medium.com/deno-the-complete-reference/using-logger-in-deno-44c5b2372bf3#91df
// ##### Note: define handlers
handlers: {
console: new log.handlers.ConsoleHandler("DEBUG", {
formatter: "{datetime} {levelName} {msg}",
}),
file: new log.handlers.RotatingFileHandler("INFO", {
filename: "./logs/process.log",
maxBytes: 15,
maxBackupCount: 5,
formatter: (rec) =>
JSON.stringify({
region: rec.loggerName,
ts: rec.datetime,
level: rec.levelName,
data: rec.msg,
}),
}),
},
// ##### Note: assign handlers to loggers
loggers: {
default: {
level: "DEBUG", // ##### Note: Pre-fix for process.js (terminal display)
handlers: ["console"],
},
client: {
level: "INFO",
handlers: ["file"],
},
},
};
// ############################################################
// ##### Information for console debugging purposes (END) #####
// ############################################################
// ########################################################
// ##### sources of API's to collect Abstract (START) #####
// ########################################################
export const abstractSources = [
{
name: "crossref", // ##### Note: CrossRef
wikidataProperty: {
id: "P356", // ##### Note: Property number in Wikidata
label: "DOI",
},
url: (id) => `https://api.crossref.org/v1/works/${id}`,
path: "message.abstract",
format: "json",
},
{
name: "pmc", // ##### Note: PubMedCentral
wikidataProperty: {
id: "P932", // ##### Note: Property number in Wikidata
label: "PMCID",
},
url: (id) =>
`https://www.ebi.ac.uk/europepmc/webservices/rest/article/PMC/PMC${id}?resultType=core&format=json`,
path: "result.abstractText",
format: "json",
},
{
name: "pubmed", // ##### Note: PubMed
wikidataProperty: {
id: "P698", // ##### Note: Property number in Wikidata
label: "PMID",
},
url: (id) =>
`https://www.ebi.ac.uk/europepmc/webservices/rest/article/MED/${id}?resultType=core&format=json`,
path: "result.abstractText",
format: "json",
},
];
console.log(filename_this + "Finished reading commands from this script for processing!");
// ######################################################
// ##### sources of API's to collect Abstract (END) #####
// ######################################################