-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathslackex2html.sh
executable file
·326 lines (308 loc) · 11.2 KB
/
slackex2html.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
#!/usr/bin/env bash
# Source the config.
if [ -f ~/.slackex.cfg ]; then
. ~/.slackex.cfg
elif [ -f .slackex.cfg ]; then
. .slackex.cfg ]
else
echo "No config (.slackex.cfg) found"
exit 1
fi
if ! [ -z $1 ]; then
if echo "$1" | grep -qis "\.zip$"; then
echo "Usuing $1 as archive"
SRCDIR=$(mktemp -d)
echo "Using $WORKDIR for archive extraction"
unzip "$1" -d $SRCDIR || die "$?"
else
echo " I don't know what to do with $@"
echo " Usage:"
echo " slackex2html [ archive-name.zip ]"
exit
fi
fi
# make sure the java script and .css are in the Destination Directory
cp *.css *.js $DSTDIR
# Setup the Header for the main index.
# This includes the js bits needed for the search to work.
INDEXHEAD="<html><head> <title>vCISO Catalyst Slack History</title>
<meta charset=\"utf-8\" />
<script src=\"lunr.js\"></script>
<link href=\"search.css\" rel=\"stylesheet\" type=\"text/css\">
<script src=\"lunr_index.js\"></script>
<script src=\"lunrclient.js\"></script>
</head>
<body>
<img src=\"$LLOGO\" height=\"100\">
<h1>$ORG Slack History</h1>
<p>Enter a search term, e.g., GRC</p>
<form id=\"lunrSearchForm\" name=\"lunrSearchForm\">
<input class=\"search-input\" name=\"q\" placeholder=\"Enter search term\" type=\"text\"> <input type=\"submit\" value=\"Search\">
</form>
<div class=\"loading\" id=\"loading-div\">
<i class=\"fa fa-spinner fa-pulse fa-3x fa-fw\"></i>
<span class=\"sr-only\">Loading...</span>
</div>
<p>Search results will be written below:</p>
<div class=\"resultCount\" id=\"resultCount\"></div>
<div id=\"searchResults\"></div>
<b> or </b>Browse by Channel</p>"
# Let's get started.
mkdir -p $DSTDIR
# This is where we start. It should be the root of the archive from Slack.
if [ -z $SRCDIR ]; then
SRCDIR=$(pwd)
fi
# Pre run Cleanup
if [ -f slackex2html.debug ]; then
rm $SRCDIR/slackex2html.debug
fi
# Check to see if we are running on MacOS or Linux, then do some setup.
OS=$(uname)
if [ "$OS" == "Darwin" ]; then
jq=$(which jq)
if [ "x$jq" == "x" ]; then
echo "Looks like you are on a Mac, but dont' have jq"
echo "You need to use brew to install jq"
exit
fi
Date() {
date -r $( jq -r ".[$INDEX]|.ts" $INPUT | sed "s/\(^[0-9]*\)\..*$/\1/") +%T
}
else
jq=$(which jq)
if [ "x$jq" == "x" ]; then
echo "The jq utility was not found."
echo "Use your distribution's package manager to install jq"
exit
fi
Date() {
date -d @$( jq -r ".[$INDEX]|.ts" $INPUT) +%T
}
fi
# Setup a debug function..
Debug() {
if [ $DEBUG -eq 1 ]; then
echo -e "debug:\n$1" | tee -a $SRCDIR/slackex2html.debug
fi
}
# Use this for Footer
FOOTER="<p>Generated by <a href=\"https://github.com/immauss/SlackEx2HTML\">SlackEx2HTML</a></p><img src=\"$SLOGO\" HEIGHT=\"50\""
# First we need to create the destination structure.
cd $SRCDIR
SRCDIRS=$(find . -mindepth 1 -type d | sed "s/^\.\///" | sort )
# then create the new structure in the destination
cd $DSTDIR
for DIR in $SRCDIRS; do
mkdir -p $DIR 2> /dev/null
done
# Create an associative array for the user name to ID mapping.
Debug " $SRCDIR/users.json"
URECORDS=$( jq ".|length" $SRCDIR/users.json)
INDEX=0
declare -A Users
echo "Reading Users into array for replacements later"
USERCOUNT=0
while [ $INDEX -lt $URECORDS ]; do
ID=$( jq -r ".[$INDEX].id" $SRCDIR/users.json )
NAME=$( jq -r ".[$INDEX].real_name" $SRCDIR/users.json )
Debug "$ID,$NAME"
INDEX=$( expr $INDEX + 1 )
Users[$ID]="$NAME"
USERCOUNT=$( expr $USERCOUNT + 1 )
if [ $(expr $USERCOUNT % 10) -eq 0 ]; then
echo -n "."
fi
done
echo
# Process each file. The final dir structure will mirror the archive's structure,
# but include sub folders for the year under each channel folder.
# Files in each folder will be by month.
#
echo "Now processing Channels"
for DIR in $SRCDIRS; do
echo -e "\nChannel #$DIR"
cd ${SRCDIR}/${DIR}
Debug "PWD $(pwd)"
for MONTH in 01 02 03 04 05 06 07 08 09 10 11 12; do
Debug "MONTH $MONTH"
for INPUT in $( ls ????-$MONTH-??.json 2> /dev/null| sort); do
Debug "INPUT at Start -$INPUT-"
DATE=$( echo $INPUT | sed "s/.json$//")
YEAR=$(echo $DATE | sed "s/\(^....\)-.*$/\1/")
mkdir -p "${DSTDIR}/${DIR}/${YEAR}"
OUTPUT="${DSTDIR}/${DIR}/${YEAR}/${MONTH}.html"
Title="$ORG #$DIR $YEAR-$MONTH"
if ! [ -f "$OUTPUT" ]; then
echo "<html><head> <title>$Title</title></head><body> " > $OUTPUT
echo "<center><h1>#$DIR</h1></center>" >> $OUTPUT
echo "xxxNAVIGATIONxxx" >> $OUTPUT
fi
Debug "MONTH $MONTH\n YEAR $YEAR\n OUTPUT $OUTPUT\n PWD $(pwd)"
# Setup the initial Table.
echo "<h1>$DATE</h1><table>" >> $OUTPUT
RECORDS=$(jq '.|length' $INPUT) # File record count (number of messages in the file)
LASTUSER="null"
# process all the records in the file.
INDEX=0
Debug "OUTPUT $OUTPUT\n SRCDIR $SRCDIR\n INPUT $INPUT\n DATE $DATE\n YEAR $YEAR\n PWD $(pwd)"
while [ $INDEX -lt $RECORDS ]; do
if [ $(expr $INDEX % 100) -eq 0 ]; then
echo -n "."
fi
TS=$( Date )
MSG=$( jq -r ".[$INDEX]|.text" $INPUT | tr '<|>|\r' ' ')
# Replace the UIDs with actual user names on mentions
while echo "$MSG" | tr -d "\n" | egrep -sq "[@\b]U[A-Z0-9]{10}\b" ; do
ID=$(echo "$MSG" | tr -d "\n" | sed -E "s/^.*(U[A-Z0-9]{10}).*$/\1/")
if [ "${Users[$ID]}x" != "x" ]; then
Debug "ID: $ID User: \"${Users[$ID]}\"\nMSG: $MSG"
MSG=$(echo "$MSG" | sed "s/${ID}/${Users[$ID]}/g")
if [ $? -ne 0 ]; then
Debug "MSG: $MSG\nID: $ID\nUser: \"${Users[$ID]}\""
fi
else
if [ ${#ID} -eq 11 ]; then
MSG=$(echo "$MSG" | sed "s/${ID}/UNKNOWN/g")
Debug "##NO User Match ##\nMSG: \"$MSG\"\nID: $ID\nUser: ${Users[$ID]}"
fi
if [ $? -ne 0 ]; then
Debug "##NO User Match ##\nsed command failed.\nMSG: \"$MSG\"\nID: $ID\nUser: ${Users[$ID]}"
fi
fi
Debug "Found $ID in MSG:\n$MSG\nFrom: $INPUT at INDEX: $INDEX"
done
# This sets up the Username and avatar, but sets them to null if the last
# entry was from the same user. This gives a cleaner look in the rendered html
USER=$( jq -r ".[$INDEX]|.user_profile.real_name" $INPUT)
Debug "LAST $LASTUSER Current $USER USERNAME $USERNAME"
if [ "$LASTUSER" != "$USER" ]; then
USERNAME="$USER"
AVATAR="<img src=\"$( jq -r ".[$INDEX]|.user_profile.image_72" $INPUT)\">"
else
USERNAME=""
AVATAR=""
fi
LASTUSER=$USER
Debug "$TS\n$USER\n$MSG\n$AVATAR"
if [ "$USERNAME" == "null" ]; then
USERNAME=""
AVATAR=""
fi
# Create a new row in the table;
echo "<tr><td style=\"vertical-align: top\"><b>${USERNAME}</b></td><td style=\"vertical-align: top\">$AVATAR</td><td style=\"vertical-align: top\">$TS:</td><td style=\"vertical-align: top\"> $MSG</td></tr>" >> $OUTPUT
# increase the INDEX
INDEX=$( expr $INDEX + 1)
done
Debug "Close table INPUT $INPUT OUTPUT $OUTPUT"
if [ -f $OUTPUT ]; then
echo "</table>" >> $OUTPUT
fi
done
Debug "Close tag INPUT $INPUT \n OUTPUT $OUTPUT"
# We first check to see if the file exists as it may not.
# This happens when there is no source file for the months in this loop.
if [ -f "$OUTPUT" ]; then
if [ $( grep -c xxxNAVIGATIONxxx $OUTPUT) -lt 2 ]; then
echo "xxxNAVIGATIONxxx" >> $OUTPUT
echo $FOOTER >> $OUTPUT
echo "</body></html>" >> $OUTPUT
fi
fi
done
done
# just to make the screen output prettier.
echo
# Now we need to create some index files to make the generated HTML easier to navigate.
# Starts at Index of channels
# In each channel folder, there is an index wich lists:
# YEAR:
# Jan Feb Mar
# Apr May Jun
# Jul Aug Sep
# Oct Nov Dec
# Each month links to that months messages
# at top and bottom of each page, navigation to prev & next in
# chronological order.
# Array for Month names
# Element 0 is NULL because there is no Month 0, this make the element number align with the
# numerical month.
MONTHA=("NULL" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec")
# Make sure we are in the destination directory
cd $DSTDIR
# #####################
# Create Channel index.
#######################
echo "$INDEXHEAD" > index.html
echo "<h1>$ORGNAME Slack History</h1>" >> index.html
for Channel in $( ls -d -1 */ | tr -d "/"); do
echo "<a href=\"$Channel/index.html\">#$Channel</a><br>" >> index.html
done
echo $FOOTER >> index.html
echo "</body></html>" >> index.html
# Process through channels
for DIR in $(ls -d -1 */); do
cd $DSTDIR/$DIR
echo "Working in $(pwd)"
Channel=$( echo $DIR | tr -d "/")
# First create the Navigation from prev <-> Next.
files=($(find . -name "*.html" | grep -v index.html |sort -n))
for (( i=0; i<${#files[@]}; i++ )); do
filename="${files[$i]}";
if (( $i > 0 )); then
prevfile="${files[$i-1]}"
PREV="<a href=\"$RELDIR/$DIR/$prevfile\">Previous</a>"
else
PREV=""
fi
if (( $i < ${#files[@]}-1 )); then
nextfile="${files[$i+1]}"
NEXT="<a href=\"$RELDIR/$DIR/$nextfile\">Next</a>"
else
NEXT=""
fi
# Insert Navigation
NAVIGATION="<a href=\"$HOME\">Home</a></br><a href=\"$RELDIR/index.html\">Channel Index</a> </p>/$PREV --- $NEXT</br>"
Debug "$filename\n$NAVIGATION\n$(pwd)\n$DIR\n$DSTDIR\n$(ls -l)\n$(ls -l $filename)"
sed -i "s|xxxNAVIGATIONxxx|$NAVIGATION|g" $filename
done
# Now create the channel index file based on available month files.
# filenames look like: "./2023/04.html"
echo "<html> <head> <title>$ORGNAME #$Channel Index</title></head><body>" > index.html
echo "<center><h2>Index of <i>#$Channel</i> at $ORGNAME</h2></center>" >> index.html
echo "<a href=\"$HOME\">Home</a></br><a href=\"$RELDIR/index.html\">Channel Index</a>" >> index.html
declare -A Links
for (( i=0; i<${#files[@]}; i++ )); do
filename="${files[$i]}";
YEAR=$( echo $filename | sed "s/^\.\/\(....\)\/..\.html$/\1/")
YEARS=(${YEARS[@]} "$YEAR")
MONTH=${MONTHA[$(echo $filename | sed "s/^\.\/....\/\(..\)\.html$/\1/;s/^0//")]}
Links[$YEAR-$MONTH]="<a href=\"$filename\">$MONTH</a>"
done
# Make YEARS array unique so we can use it for table creation.
YEARS=($(for x in "${YEARS[@]}"; do echo "${x}"; done | sort -ru))
for Year in ${YEARS[@]}; do
for Month in ${MONTHA[@]}; do
#echo "$DIR $Year $Month ${Links[$Year-$Month]}"
if [ "x${Links[$Year-$Month]}" == "x" ]; then
Links[$Year-$Month]="$Month"
fi
done
echo "<h3>$Year</h3>" >> index.html
echo "<table border="1">" >> index.html
echo "<tr><td>${Links[${Year}-Jan]}</td><td>${Links[${Year}-Feb]}</td><td>${Links[${Year}-Mar]}</td></tr>" >> index.html
echo "<tr><td>${Links[${Year}-Apr]}</td><td>${Links[${Year}-May]}</td><td>${Links[${Year}-Jun]}</td></tr>" >> index.html
echo "<tr><td>${Links[${Year}-Jul]}</td><td>${Links[${Year}-Aug]}</td><td>${Links[${Year}-Sep]}</td></tr>" >> index.html
echo "<tr><td>${Links[${Year}-Oct]}</td><td>${Links[${Year}-Nov]}</td><td>${Links[${Year}-Dec]}</td></tr>" >> index.html
echo "</table>" >> index.html
done
echo $FOOTER >> index.html
echo "</body></html>" >> index.html
# Make sure these are clean for next run
YEARS=()
Links=()
done
# Rebuild the index
cd $DSTDIR
npm install lunr cheerio
node build_index.js