-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathmantis2trac.py
1077 lines (952 loc) · 44.3 KB
/
mantis2trac.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Import Mantis bugs into a Trac database.
Requires: Trac 0.9.X or newer from http://trac.edgewall.com/
Python 2.4 from http://www.python.org/
MySQL >= 3.23 from http://www.mysql.org/
Example use:
python mantis2trac.py --db mantis --tracenv /usr/local/trac-projects/myproj/ \
--host localhost --user root --clean --append --products foo,bar
Version 1.6
Author: Steffen Mecke (stm2@users.sourceforge.net)
Date: January 8, 2015
Version 1.5
Author: Matthew Parmelee (mparmelee@interworx.com)
Date: July 16, 2013
Version 1.4
Author: John Lichovník (licho@ufo.cz)
Date: 10.9.2007
Version 1.3
Author: Anton Stroganov (stroganov.a@gmail.com)
Date: December 19, 2006
Based on version 1.1 from:
Author: Joao Prado Maia (jpm@pessoal.org)
Based on version 1.0 from:
Paul Baranowski (paul@paulbaranowski.org)
Based on bugzilla2trac.py by these guys (thank you!):
Dmitry Yusupov <dmitry_yus@yahoo.com> - bugzilla2trac.py
Mark Rowe <mrowe@bluewire.net.nz> - original TracDatabase class
Bill Soudan <bill@soudan.net> - Many enhancements
Changes in version 1.6:
- allow to append to an existing project (with correct id mapping)
- fixed to work with mysql by removing INSERT OR REPLACE syntax
Changes in version 1.5:
- repaired queries to be consistent with Mantis updates
- corrected timestamp conversion
- enabled file attachment migration
Changes in version 1.4:
- fixed strftime for Python 2.4
- fixed Mantis text_id in ticket and comment queries (original version was sometimes adding mismatched descriptions and comments)
- added IGNORE_VERSION switch
Changes since version 1.2:
- better join in the attachment author finding query
- changed default encoding to be utf8
- added working status->keyword migration for statuses that don't have exact Trac equivalents
Changes since version 1.1:
- Made it work against Trac running on MySQL (specifically, changes to the
LAST_INSERT_ID() call on line 382 (in the addTicket function))
- Couple of bugfixes
- Works fine against 10.2
- Modified to allow specifying product list on command line
- Modified to migrate database-stored mantis attachments correctly.
Nota Bene!!! The script requires write access to the attachments
directory of the trac env. So, suggested sequence of actions:
- chmod -R 777 /usr/local/trac-projects/myproj/attachments/
- run the script
- chown -R apache /usr/local/trac-projects/myproj/attachments/
- chgrp -R webuser /usr/local/trac-projects/myproj/attachments/
- chmod -R 755 /usr/local/trac-projects/myproj/attachments/
Changes since version 1.0:
- Made it to work against Trac 0.9.3 (tweaks to make the Environment class work)
- Re-did all prepared statements-like queries to avoid a DB error
- Fixed a reference to the wrong variable name when adding a comment
Notes:
- Private bugs will become public
- Some ticket changes will not be preserved since they have no
equivalents in Trac.
- I consider milestones and versions to be the same thing (actually,
I dont really care about the version, because for our project, bugs are
only in the 'previous version').
- Importing attachments is not implemented (couldnt get it to work,
and we didnt have enough attachments to justify spending time on this)
"Clean" will not delete your existing attachments. There is code in here
to support adding attachments, but you will have to play with it to
make it work. If you search for the word "attachment" you will find
all the code related to this.
- Ticket descriptions & comments will be re-wrapped to 70 characters.
This may mess up your formatting for your bugs. If you dont want to do
this, search for textwrap.fill() and fix it.
- You will probably want to change "report.css" in trac to handle one more
level of priorities (default trac has 6 levels of priorities, while Mantis
has 7). When you look at your reports, the color schemes will look wrong.
The lines that control the priority color scheme look like this:
#tktlist tr.color1-odd { background: #fdc; border-color: #e88; color: #a22 }
#tktlist tr.color1-even { background: #fed; border-color: #e99; color: #a22 }
I added a new level 2 ("urgent") with an orange color,
and incremented all the rest of the levels:
#tktlist tr.color2-odd { background: #FFE08F; border-color: #e88; color: #a22 }
#tktlist tr.color2-even { background: #FFE59F; border-color: #e99; color: #a22 }
"""
from urllib import quote
from datetime import datetime, date
import time
import hashlib
###
### Conversion Settings -- edit these before running if desired
###
# Mantis version.
#
# Currently, the following mantis versions are known to work:
# 0.19.X
#
# If you run this script on a version not listed here and it is successful,
# please report it to the Trac mailing list so we can update the list.
MANTIS_VERSION = '0.19'
# MySQL connection parameters for the Mantis database. These can also
# be specified on the command line.
MANTIS_DB = 'mantis'
MANTIS_HOST = 'localhost'
MANTIS_USER = 'root'
MANTIS_PASSWORD = ''
# Path to the Trac environment.
TRAC_ENV = '/var/www/trac/projectname/'
# If true, all existing Trac tickets will be removed
# prior to import.
TRAC_CLEAN = False
# If TRAC_CLEAN is true and this is true, tickets will be appended
TRAC_APPEND = True
# Enclose imported ticket description and comments in a {{{ }}}
# preformat block? This formats the text in a fixed-point font.
PREFORMAT_COMMENTS = False
# Products are now specified on command line.
# By default, all bugs are imported from Mantis. If you add a list
# of products here, only bugs from those products will be imported.
# Warning: I have not tested this script where this field is blank!
# default products to ignore:
PRODUCTS = [ ]
# Trac doesn't have the concept of a product. Instead, this script can
# assign keywords in the ticket entry to represent products.
#
# ex. PRODUCT_KEYWORDS = { 'product1' : 'PRODUCT1_KEYWORD' }
PRODUCT_KEYWORDS = {}
# Bug comments that should not be imported. Each entry in list should
# be a regular expression.
IGNORE_COMMENTS = [
# '^Created an attachment \(id='
]
# Ticket changes in Trac have the restriction where the
# bug ID, field, and time must be unique for all entries in the ticket
# changes table.
# Mantis, for unknown reasons, has fields that can change two states
# in under a second (e.g. "milestone":""->"1.0", "milestone":"1.0"->"2.0").
# Setting this to true will attempt to fix these cases by adjusting the
# time for the 2nd change to be one second more than the original time.
# I dont know why you'd want to turn this off, but I give you the option
# anyhow. :)
TIME_ADJUSTMENT_HACK = True
# If set to true, version numbers wont be assigned to tickets (just milestones)
IGNORE_VERSION = False
###########################################################################
### You probably don't need to change any configuration past this line. ###
###########################################################################
# Mantis status to Trac status translation map.
#
# NOTE: bug activity is translated as well, which may cause bug
# activity to be deleted (e.g. resolved -> closed in Mantis
# would translate into closed -> closed in Trac, so we just ignore the
# change).
#
# Possible Trac 'status' values: 'new', 'assigned', 'reopened', 'closed'
STATUS_TRANSLATE = {
10 : 'new', # 10 == 'new' in mantis
20 : 'assigned', # 20 == 'feedback'
30 : 'new', # 30 == 'acknowledged'
40 : 'new', # 40 == 'confirmed'
50 : 'assigned', # 50 == 'assigned'
60 : 'assigned', # 60 == 'QA'
80 : 'closed', # 80 == 'resolved'
90 : 'closed' # 90 == 'closed'
}
# Unused:
# Translate Mantis statuses into Trac keywords. This provides a way
# to retain the Mantis statuses in Trac. e.g. when a bug is marked
# 'verified' in Mantis it will be assigned a VERIFIED keyword.
# STATUS_KEYWORDS = {
# 'confirmed' : 'CONFIRMED',
# 'feedback' : 'FEEDBACK',
# 'acknowledged':'ACKNOWLEDGED',
# 'QA':'QA'
# }
STATUS_KEYWORDS = {
20 : 'FEEDBACK',
30 : 'ACKNOWLEDGED',
40 : 'CONFIRMED',
60 : 'QA',
80 : 'RESOLVED'
}
# Possible Trac resolutions are 'fixed', 'invalid', 'wontfix', 'duplicate', 'worksforme'
RESOLUTION_TRANSLATE = {
10 : '', # 10 == 'open' in mantis
20 : 'fixed', # 20 == 'fixed'
30 : '', # 30 == 'reopened' (TODO: 'reopened' needs to be mapped to a status event)
40 : 'invalid', # 40 == 'unable to duplicate'
50 : 'wontfix', # 50 == 'not fixable'
60 : 'duplicate', # 60 == 'duplicate'
70 : 'invalid', # 70 == 'not an issue'
80 : '', # 80 == 'suspended'
90 : 'wontfix', # 90 == 'wont fix'
}
# Mantis severities (which will also become equivalent Trac severities)
##SEVERITY_LIST = (('block', '80'),
## ('crash', '70'),
## ('major', '60'),
## ('minor', '50'),
## ('tweak', '40'),
## ('text', '30'),
## ('trivial', '20'),
## ('feature', '10'))
SEVERITY_LIST = (('block', '1'),
('crash', '2'),
('major', '3'),
('minor', '4'),
('tweak', '5'),
('text', '6'),
('trivial', '7'),
('feature', '8'))
# Translate severity numbers into their text equivalents
SEVERITY_TRANSLATE = {
80 : 'block',
70 : 'crash',
60 : 'major',
50 : 'minor',
40 : 'tweak',
30 : 'text',
20 : 'trivial',
10 : 'feature'
}
# Mantis priorities (which will also become Trac priorities)
##PRIORITY_LIST = (('immediate', '60'),
## ('urgent', '50'),
## ('high', '40'),
## ('normal', '30'),
## ('low', '20'),
## ('none', '10'))
PRIORITY_LIST = (('immediate', '1'),
('urgent', '2'),
('high', '3'),
('normal', '4'),
('low', '5'),
('none', '6'))
# Translate priority numbers into their text equivalent
PRIORITY_TRANSLATE = {
60 : 'immediate',
50 : 'urgent',
40 : 'high',
30 : 'normal',
20 : 'low',
10 : 'none'
}
# Some fields in Mantis do not have equivalents in Trac. Changes in
# fields listed here will not be imported into the ticket change history,
# otherwise you'd see changes for fields that don't exist in Trac.
IGNORED_ACTIVITY_FIELDS = ['', 'project_id', 'reproducibility', 'view_state', 'os', 'os_build', 'duplicate_id']
###
### Script begins here
###
import os
import re
import sys
import string
import StringIO
import MySQLdb
import MySQLdb.cursors
from trac.env import Environment
if not hasattr(sys, 'setdefaultencoding'):
reload(sys)
sys.setdefaultencoding('utf-8')
# simulated Attachment class for trac.add
# unused in 1.2
class Attachment:
def __init__(self, name, data):
self.filename = name
self.file = StringIO.StringIO(data.tostring())
# simple field translation mapping. if string not in
# mapping, just return string, otherwise return value
class FieldTranslator(dict):
def __getitem__(self, item):
if not dict.has_key(self, item):
return item
return dict.__getitem__(self, item)
statusXlator = FieldTranslator(STATUS_TRANSLATE)
class TracDatabase(object):
def __init__(self, path, append):
self.append = _append
self.env = Environment(path)
self._db = self.env.get_db_cnx()
self._db.autocommit = False
self.loginNameCache = {}
self.fieldNameCache = {}
def db(self):
return self._db
def hasTickets(self):
c = self.db().cursor()
c.execute('''SELECT count(*) FROM ticket''')
return int(c.fetchall()[0][0]) > 0
def assertNoTickets(self):
if not self._append or self.hasTickets():
raise Exception("Will not modify database with existing tickets!")
return
def setSeverityList(self, s):
"""Remove all severities, set them to `s`"""
self.assertNoTickets()
c = self.db().cursor()
c.execute("""DELETE FROM enum WHERE type='severity'""")
for value, i in s:
print "inserting severity ", value, " ", i
c.execute("""INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)""",
("severity", value.encode('utf-8'), i,))
self.db().commit()
def setPriorityList(self, s):
"""Remove all priorities, set them to `s`"""
self.assertNoTickets()
c = self.db().cursor()
c.execute("""DELETE FROM enum WHERE type='priority'""")
for value, i in s:
print "inserting priority ", value, " ", i
c.execute("""INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)""",
("priority", value.encode('utf-8'), i,))
self.db().commit()
def setComponentList(self, l, key):
"""Remove all components, set them to `l`"""
self.assertNoTickets()
c = self.db().cursor()
c.execute("""DELETE FROM component""")
for comp in l:
print "inserting component '",comp[key],"', owner", comp['owner']
c.execute("""INSERT INTO component (name, owner) VALUES (%s, %s)""",
(comp[key].encode('utf-8'), comp['owner'].encode('utf-8'),))
self.db().commit()
def setVersionList(self, v, key):
"""Remove all versions, set them to `v`"""
self.assertNoTickets()
c = self.db().cursor()
c.execute("""DELETE FROM version""")
for vers in v:
print "inserting version ", vers[key]
c.execute("""INSERT INTO version (name) VALUES (%s)""",
(vers[key].encode('utf-8'),))
self.db().commit()
def setMilestoneList(self, m, key):
"""Remove all milestones, set them to `m`"""
self.assertNoTickets()
c = self.db().cursor()
c.execute("""DELETE FROM milestone""")
for ms in m:
print "inserting milestone ", ms[key]
c.execute("""INSERT INTO milestone (name, due, completed) VALUES (%s, %s, %s)""",
(ms[key].encode('utf-8'), self.convertTime(ms['date_order']), ms['released']))
self.db().commit()
def addTicket(self, id, time, changetime, component,
severity, priority, owner, reporter, cc,
version, milestone, status, resolution,
summary, description, keywords):
c = self.db().cursor()
if IGNORE_VERSION:
version=''
desc = description
type = 'defect'
if component == 'Features' or severity == 'feature':
type = 'enhancement'
if PREFORMAT_COMMENTS:
desc = '{{{\n%s\n}}}' % desc
print "inserting ticket %s -- \"%s\"" % (id, summary[0:40].replace("\n", " "))
c.execute("""INSERT INTO ticket (type, time, changetime, component,
severity, priority, owner, reporter, cc,
version, milestone, status, resolution,
summary, description, keywords)
VALUES (%s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s, %s,
%s, %s, %s)""",
(type, self.convertTime(time), self.convertTime(changetime), component.encode('utf-8'),
severity.encode('utf-8'), priority.encode('utf-8'), owner, reporter, cc,
version, milestone.encode('utf-8'), status.lower(), resolution,
summary.decode('utf-8'), desc, keywords.encode('utf-8')))
self.db().commit()
## TODO: add database-specific methods to get the last inserted ticket's id...
## PostgreSQL:
# c.execute('''SELECT currval("ticket_id_seq")''')
## SQLite:
# c.execute('''SELECT last_insert_rowid()''')
## MySQL:
# c.execute('''SELECT LAST_INSERT_ID()''')
# Oh, Trac db abstraction layer already has a function for this...
return self.db().get_last_id(c,'ticket')
def convertTime(self,time2):
time2 = datetime.fromtimestamp(time2)
return long(str(int(time.mktime(time2.timetuple()))) + '000000')
def addTicketComment(self, ticket, time, author, value):
#return
print " * adding comment \"%s...\"" % value[0:40]
comment = value
if PREFORMAT_COMMENTS:
comment = '{{{\n%s\n}}}' % comment
c = self.db().cursor()
c.execute("""INSERT INTO ticket_change (ticket, time, author, field, oldvalue, newvalue)
VALUES (%s, %s, %s, %s, %s, %s)""",
(ticket, self.convertTime(time), author, 'comment', '', comment))
self.db().commit()
def addTicketChange(self, ticket, time, author, field, oldvalue, newvalue):
if (field[0:4]=='doba'):
return
if field == 'milestone':
field = 'product_version'
print " * adding ticket change \"%s\": \"%s\" -> \"%s\" (%s)" % (field, oldvalue[0:20], newvalue[0:20], time)
c = self.db().cursor()
#workaround 'unique' ticket_change warnings POTENTIALLY BAD IDEA ALERT
sql = "SELECT * FROM ticket_change WHERE field='%s' AND ticket=%s AND time=%s" % (field, ticket, self.convertTime(time))
c.execute(sql)
fixtime = c.fetchall()
if fixtime:
time = time + 1
c.execute("""INSERT INTO ticket_change (ticket, time, author, field, oldvalue, newvalue)
VALUES (%s, %s, %s, %s, %s, %s)""",
(ticket, self.convertTime(time), author, field, oldvalue.encode('utf-8'), newvalue.encode('utf-8')))
self.db().commit()
#workaround 'unique' ticket warnings POTENTIALLY BAD IDEA ALERT
sql = "SELECT * FROM ticket WHERE %s='%s' AND id=%s AND time=%s" % (field, newvalue, ticket, self.convertTime(time))
c.execute(sql)
fixtime = c.fetchall()
if fixtime:
time = time + 1
# Now actually change the ticket because the ticket wont update itself!
sql = "UPDATE ticket SET %s='%s' WHERE id=%s" % (field, newvalue, ticket)
c.execute(sql)
self.db().commit()
# unused in 1.2
def addAttachment(self, id, attachment, description, author):
print 'inserting attachment for ticket %s -- %s' % (id, description)
attachment.filename = attachment.filename.encode('utf-8')
self.env.create_attachment(self.db(), 'ticket', str(id), attachment, description.encode('utf-8'),
author, 'unknown')
def getLoginName(self, cursor, userid):
if userid not in self.loginNameCache and userid is not None and userid != '':
cursor.execute("SELECT username,email,realname,last_visit FROM mantis_user_table WHERE id = %i" % int(userid))
result = cursor.fetchall()
if result:
loginName = result[0]['username']
print 'Adding user %s to sessions table' % loginName
c = self.db().cursor()
# check if user is already in the sessions table
c.execute("SELECT sid FROM session WHERE sid = '%s'" % result[0]['username'].encode('utf-8'))
r = c.fetchall()
# if there was no user sid in the database already
if not r:
sessionSql = """INSERT INTO session
(sid, authenticated, last_visit)
VALUES (%s, %s, %d)""", (result[0]['username'].encode('utf-8'), '1', self.convertTime(result[0]['last_visit']))
# pre-populate the session table and the realname/email table with user data
try:
c.execute(sessionSql)
except:
print 'failed executing sql: '
print sessionSql
print 'could not insert %s into sessions table: sql error ' % (loginName)
self.db().commit()
# insert the user's real name into session attribute table
try:
c.execute(
"""INSERT INTO session_attribute
(sid, authenticated, name, value)
VALUES
(%s, %s, %s, %s)""", (result[0]['username'].encode('utf-8'), '1', 'name', result[0]['realname'].encode('utf-8')))
except:
print 'failed executing session-attribute sql'
self.db().commit()
# insert the user's email into session attribute table
try:
c.execute(
"""INSERT INTO session_attribute
(sid, authenticated, name, value)
VALUES
(%s, %s, %s, %s)""", (result[0]['username'].encode('utf-8'), '1', 'email', result[0]['email'].encode('utf-8')))
except:
print 'failed executing session-attribute sql2'
self.db().commit()
else:
print 'warning: unknown mantis userid %d, recording as anonymous' % userid
loginName = ''
self.loginNameCache[userid] = loginName
elif userid is None or userid == '':
self.loginNameCache[userid] = ''
return self.loginNameCache[userid]
def get_attachments_dir(self,bugid=0):
if bugid > 0:
return 'importfiles/%i/' % bugid
else:
return 'importfiles/'
def _mkdir(newdir):
"""works the way a good mkdir should :)
- already exists, silently complete
- regular file in the way, raise an exception
- parent directory(ies) does not exist, make them as well
"""
if os.path.isdir(newdir):
pass
elif os.path.isfile(newdir):
raise OSError("a file with the same name as the desired " \
"dir, '%s', already exists." % newdir)
else:
head, tail = os.path.split(newdir)
if head and not os.path.isdir(head):
_mkdir(head)
#print "_mkdir %s" % repr(newdir)
if tail:
os.mkdir(newdir)
def productFilter(fieldName, products):
first = True
result = ''
for product in products:
if not first:
result += " or "
first = False
result += "%s = '%s'" % (fieldName, product)
return result
def convert(_db, _host, _user, _password, _env, _force, _append):
activityFields = FieldTranslator()
# account for older versions of mantis
if MANTIS_VERSION == '0.19':
print 'Using Mantis v%s schema.' % MANTIS_VERSION
activityFields['removed'] = 'oldvalue'
activityFields['added'] = 'newvalue'
# init Mantis environment
print "Mantis MySQL('%s':'%s':'%s':'%s'): connecting..." % (_db, _host, _user, _password)
mysql_con = MySQLdb.connect(host=_host,
user=_user, passwd=_password, db=_db, compress=1,
cursorclass=MySQLdb.cursors.DictCursor, use_unicode=1)
mysql_cur = mysql_con.cursor()
# init Trac environment
print "Trac database('%s'): connecting..." % (_env)
trac = TracDatabase(_env, _append)
# force mode...
if _force == 1:
print "cleaning all tickets..."
c = trac.db().cursor()
sql = """DELETE FROM ticket_change"""
c.execute(sql)
trac.db().commit()
sql = """DELETE FROM ticket"""
c.execute(sql)
trac.db().commit()
sql = """DELETE FROM ticket_custom"""
c.execute(sql)
trac.db().commit()
sql = """DELETE FROM attachment"""
c.execute(sql)
os.system('rm -rf %s' % trac.get_attachments_dir())
os.mkdir(trac.get_attachments_dir())
trac.db().commit()
print
print '0. Finding project IDs...'
sql = "SELECT id, name FROM mantis_project_table"
if PRODUCTS:
sql += " WHERE %s" % productFilter('name', PRODUCTS)
mysql_cur.execute(sql)
project_list = mysql_cur.fetchall()
project_dict = dict()
for project_id in project_list:
print "Mantis project name '%s' has project ID %s" % (project_id['name'], project_id['id'])
project_dict[project_id['id']] = project_id['id']
print
print "1. import severities..."
trac.setSeverityList(SEVERITY_LIST)
print
print "2. import components..."
sql = "SELECT DISTINCT name as category, user_id as owner FROM mantis_category_table, mantis_bug_table WHERE user_id=user_id GROUP BY category"
if PRODUCTS:
sql += " WHERE %s" % productFilter('project_id', project_dict)
print "sql: %s" % sql
mysql_cur.execute(sql)
components = mysql_cur.fetchall()
for component in components:
component['owner'] = trac.getLoginName(mysql_cur, component['owner'])
trac.setComponentList(components, 'category')
print
print "3. import priorities..."
trac.setPriorityList(PRIORITY_LIST)
print
print "4. import versions..."
sql = "SELECT DISTINCTROW version FROM mantis_project_version_table"
if PRODUCTS:
sql += " WHERE %s" % productFilter('project_id', project_dict)
mysql_cur.execute(sql)
versions = mysql_cur.fetchall()
trac.setVersionList(versions, 'version')
print
print "5. import milestones..."
sql = "SELECT version, date_order, released, obsolete FROM mantis_project_version_table GROUP BY version"
if PRODUCTS:
sql += " WHERE %s" % productFilter('project_id', project_dict)
mysql_cur.execute(sql)
milestones = mysql_cur.fetchall()
for milestone in milestones:
if milestone['obsolete'] != 0 or milestone['released'] != 0:
milestone['released'] = trac.convertTime(milestone['date_order'])
else:
milestone['released'] = None
trac.setMilestoneList(milestones, 'version')
print
print '6. retrieving bugs...'
sql = "SELECT mantis_bug_table.id, date_submitted, last_updated, mantis_category_table.name, severity, priority, handler_id, reporter_id, version, target_version, summary, mantis_bug_table.status, resolution, bug_text_id FROM mantis_bug_table, mantis_category_table "
sql += "WHERE mantis_bug_table.category_id=mantis_category_table.id "
sql += "GROUP BY mantis_bug_table.id "
if PRODUCTS:
sql += " WHERE %s" % productFilter('project_id', project_dict)
sql += " ORDER BY id"
mysql_cur.execute(sql)
bugs = mysql_cur.fetchall()
print
print "7. import bugs and bug activity..."
totalComments = 0
totalTicketChanges = 0
totalAttachments = 0
errors = []
timeAdjustmentHacks = []
for bug in bugs:
bugid = bug['id']
ticket = {}
keywords = []
ticket['id'] = bugid
ticket['time'] = bug['date_submitted']
ticket['changetime'] = bug['last_updated']
ticket['component'] = bug['name']
ticket['severity'] = SEVERITY_TRANSLATE[bug['severity']]
ticket['priority'] = PRIORITY_TRANSLATE[bug['priority']]
ticket['owner'] = trac.getLoginName(mysql_cur, bug['handler_id'])
ticket['reporter'] = trac.getLoginName(mysql_cur, bug['reporter_id'])
ticket['version'] = bug['version']
if IGNORE_VERSION:
ticket['version'] = ''
ticket['milestone'] = bug['version']
if bug['target_version']:
ticket['milestone'] = bug['target_version']
ticket['summary'] = bug['summary']
ticket['status'] = STATUS_TRANSLATE[bug['status']]
ticket['cc'] = ''
ticket['keywords'] = ''
# Special case for 'reopened' resolution in mantis -
# it maps to a status type in Trac.
if (bug['resolution'] == 30):
ticket['status'] = 'reopened'
ticket['resolution'] = RESOLUTION_TRANSLATE[bug['resolution']]
# Compose the description from the three text fields in Mantis:
# 'description', 'steps_to_reproduce', 'additional_information'
mysql_cur.execute("SELECT * FROM mantis_bug_text_table WHERE id = %s" % bug['bug_text_id'])
longdescs = list(mysql_cur.fetchall())
# check for empty 'longdescs[0]' field...
if len(longdescs) == 0:
ticket['description'] = ''
else:
tmpDescr = longdescs[0]['description']
if (longdescs[0]['steps_to_reproduce'].strip() != ''):
tmpDescr = ('%s\n\n=== Steps to Reproduce ===\n%s') % (tmpDescr, longdescs[0]['steps_to_reproduce'])
if (longdescs[0]['additional_information'].strip() != ''):
tmpDescr = ('%s\n\n=== Additional Information ===\n%s') % (tmpDescr, longdescs[0]['additional_information'])
ticket['description'] = tmpDescr
del longdescs[0]
# Add the ticket to the Trac database
new_id = trac.addTicket(**ticket)
print "ticket %s has id %s" % (bugid, new_id)
#
# Add ticket comments
#
mysql_cur.execute("SELECT * FROM mantis_bugnote_table, mantis_bugnote_text_table WHERE bug_id = %s AND mantis_bugnote_table.bugnote_text_id = mantis_bugnote_text_table.id ORDER BY date_submitted" % bugid)
bug_notes = mysql_cur.fetchall()
totalComments += len(bug_notes)
for note in bug_notes:
#Check for changesets, and add trac changeset links to the comments section where applicable
mysql_cur.execute("SELECT * FROM mantis_bug_history_table WHERE bug_id=%s AND date_modified > %s AND date_modified < %s AND field_name='source_changeset_attached' AND user_id=%s " % (bugid, note['date_submitted']-2, note['date_submitted']+2, note['reporter_id']))
activity = mysql_cur.fetchall()
if activity:
project, branch, commit = activity[0]['new_value'].split()
wikivalue = '%s [/browser/?rev=%s %s] [%s]' % (project, commit, branch, commit)
note['note'] = note['note'] + "\n\n" + wikivalue
trac.addTicketComment(new_id, note['date_submitted'], trac.getLoginName(mysql_cur, note['reporter_id']), note['note'])
#
# Convert ticket changes
#
mysql_cur.execute("SELECT * FROM mantis_bug_history_table WHERE bug_id=%s ORDER BY date_modified, id" % bugid)
bugs_activity = mysql_cur.fetchall()
resolution = ''
ticketChanges = []
keywords = []
for activity in bugs_activity:
field_name = activity['field_name'].lower()
# Convert Mantis field names...
# The following fields are the same in Mantis and Trac:
# - 'status'
# - 'priority'
# - 'summary'
# - 'resolution'
# - 'severity'
# - 'version'
#
# Ignore the following changes:
# - project_id
# - reproducibility
# - view_state
# - os
# - os_build
# - duplicate_id
#
# Convert Mantis -> Trac:
# - 'handler_id' -> 'owner'
# - 'fixed_in_version' -> 'milestone'
# - 'category' -> 'component'
# - 'version' -> 'milestone'
ticketChange = {}
ticketChange['ticket'] = new_id
ticketChange['oldvalue'] = activity['old_value']
ticketChange['newvalue'] = activity['new_value']
ticketChange['time'] = activity['date_modified']
ticketChange['author'] = trac.getLoginName(mysql_cur, activity['user_id'])
ticketChange['field'] = field_name
add_keywords = []
remove_keywords = []
try:
activity['old_value'] = int(activity['old_value'])
except ValueError:
activity['old_value'] = activity['old_value'].upper()
try:
activity['new_value'] = int(activity['new_value'])
except ValueError:
activity['new_value'] = activity['new_value'].upper()
if field_name == 'handler_id':
ticketChange['field'] = 'owner'
ticketChange['oldvalue'] = trac.getLoginName(mysql_cur, activity['old_value'])
ticketChange['newvalue'] = trac.getLoginName(mysql_cur, activity['new_value'])
#elif field_name == 'fixed_in_version':
#ticketChange['field'] = 'milestone'
elif field_name == 'name':
ticketChange['field'] = 'component'
elif field_name == 'version':
ticketChange['field'] = 'milestone'
elif field_name == 'status':
try:
ticketChange['oldvalue'] = STATUS_TRANSLATE[activity['old_value']]
except:
if activity['old_value'] in STATUS_TRANSLATE:
key = [k for k, v in STATUS_KEYWORDS.iteritems() if activity['old_value'] in v]
ticketChange['oldvalue'] = STATUS_TRANSLATE[key[0]]
try:
ticketChange['newvalue'] = STATUS_TRANSLATE[activity['new_value']]
except:
if activity['new_value'] in STATUS_TRANSLATE:
key = [k for k, v in STATUS_KEYWORDS.iteritems() if activity['new_value'] in v]
ticketChange['newvalue'] = STATUS_TRANSLATE[key[0]]
if activity['old_value'] in STATUS_KEYWORDS:
remove_keywords.append(STATUS_KEYWORDS[activity['old_value']])
if activity['new_value'] in STATUS_KEYWORDS:
add_keywords.append(STATUS_KEYWORDS[activity['new_value']])
elif field_name == 'priority':
ticketChange['oldvalue'] = PRIORITY_TRANSLATE[activity['old_value']]
ticketChange['newvalue'] = PRIORITY_TRANSLATE[activity['new_value']]
elif field_name == 'resolution':
ticketChange['oldvalue'] = RESOLUTION_TRANSLATE[activity['old_value']]
ticketChange['newvalue'] = RESOLUTION_TRANSLATE[activity['new_value']]
elif field_name == 'severity':
ticketChange['oldvalue'] = SEVERITY_TRANSLATE[activity['old_value']]
ticketChange['newvalue'] = SEVERITY_TRANSLATE[activity['new_value']]
elif field_name == 'source_changeset_attached':
try:
project, branch, commit = ticketChange['newvalue'].split()
wikivalue = '%s [/browser/?rev=%s %s] [%s]' % (project, commit, branch, commit)
trac.addTicketComment( ticketChange['ticket'], ticketChange['time'], ticketChange['author'], wikivalue )
except:
print
if add_keywords or remove_keywords:
# ensure removed ones are in old
old_keywords = keywords + [kw for kw in remove_keywords if kw not in keywords]
# remove from new
keywords = [kw for kw in keywords if kw not in remove_keywords]
# add to new
keywords += [kw for kw in add_keywords if kw not in keywords]
if old_keywords != keywords:
ticketChangeKw = ticketChange.copy()
ticketChangeKw['field'] = "keywords"
ticketChangeKw['oldvalue'] = ' '.join(old_keywords)
ticketChangeKw['newvalue'] = ' '.join(keywords)
ticketChanges.append(ticketChangeKw)
if field_name in IGNORED_ACTIVITY_FIELDS:
continue
# skip changes that have no effect (think translation!)
if ticketChange['oldvalue'] == ticketChange['newvalue']:
continue
ticketChanges.append (ticketChange)
totalTicketChanges += len(ticketChanges)
for ticketChange in ticketChanges:
try:
trac.addTicketChange (**ticketChange)
except:
if TIME_ADJUSTMENT_HACK:
originalTime = ticketChange['time']
ticketChange['time'] += 1
try:
trac.addTicketChange(**ticketChange)
noticeStr = " ~ Successfully adjusted time for ticket(#%s) change \"%s\": \"%s\" -> \"%s\" (%s)" % (bugid, ticketChange['field'], ticketChange['oldvalue'], ticketChange['newvalue'], ticketChange['time'])
noticeStr += "\n Original time: %s" % originalTime
timeAdjustmentHacks.append(noticeStr)
except:
errorStr = " * ERROR: unable to add ticket(#%s) change \"%s\": \"%s\" -> \"%s\" (%s)" % (bugid, ticketChange['field'], ticketChange['oldvalue'], ticketChange['newvalue'], ticketChange['time'])
errorStr += "\n The bug id, field name, and time must be unique"
errors.append(errorStr)
print errorStr
else:
errorStr = " * ERROR: unable to add ticket(#%s) change \"%s\": \"%s\" -> \"%s\" (%s)" % (bugid, ticketChange['field'], ticketChange['oldvalue'], ticketChange['newvalue'], ticketChange['time'])
errorStr += "\n The bug id, field name, and time must be unique"
errors.append(errorStr)
print errorStr
#
# Add ticket file attachments
#
attachment_sql = "SELECT b.id,b.bug_id,b.title,b.description,b.filename,b.filesize,b.file_type,b.date_added AS date_added, b.content, h.user_id FROM mantis_bug_file_table AS b LEFT JOIN mantis_bug_history_table AS h ON (h.type = 9 AND h.old_value = b.filename AND h.bug_id = b.bug_id) WHERE b.bug_id = %s" % bugid
mysql_cur.execute(attachment_sql)
attachments = mysql_cur.fetchall()
for attachment in attachments:
print attachment['user_id']
author = trac.getLoginName(mysql_cur, attachment['user_id'])
# Old attachment stuff that never worked...
# attachmentFile = open(attachment['diskfile'], 'r')
# attachmentData = attachmentFile.read()
# tracAttachment = Attachment(attachment['filename'], attachmentData)
# trac.addAttachment(bugid, tracAttachment, attachment['description'], author)
try:
try:
if(os.path.isdir(trac.get_attachments_dir(new_id)) == False):
try:
os.mkdir(trac.get_attachments_dir(new_id))
except:
errorStr = " * ERROR: couldnt create attachment directory in filesystem at %s" % trac.get_attachments_dir(new_id)
errors.append(errorStr)
print errorStr
# trac stores the files with the special characters like spaces in the filename encoded to the url
# equivalents, so we have to urllib.quote() the filename we're saving.
attachmentFile = open(trac.get_attachments_dir(new_id) + quote(attachment['filename']),'wb')
attachmentFile.write(attachment['content'])
attachmentFile.close()
except:
errorStr = " * ERROR: couldnt dump attachment data into filesystem at %s" % trac.get_attachments_dir(new_id) + attachment['filename']
errors.append(errorStr)
print errorStr
else:
attach_sql = """INSERT INTO attachment (type,id,filename,size,time,description,author,ipnr) VALUES ('ticket',%s,'%s',%i,%i,'%s','%s','127.0.0.1')""" % (new_id,attachment['filename'].encode('utf-8'),attachment['filesize'],trac.convertTime(attachment['date_added']),attachment['description'].encode('utf-8'),author)
try:
c = trac.db().cursor()
c.execute(attach_sql)
trac.db().commit()
except:
errorStr = " * ERROR: couldnt insert attachment data into database with %s" % attach_sql
errors.append(errorStr)
print errorStr
else:
print 'inserting attachment for ticket %s -- %s, added by %s' % (bugid, attachment['description'], author)
totalAttachments += 1
hash = hashlib.sha1()
hash.update(str(new_id).encode('utf-8'))
path_hash = hash.hexdigest()
hash = hashlib.sha1()
hash.update(attachment['filename'].encode('utf-8'))
file_hash = hash.hexdigest()
old_path = 'importfiles/%s/%s' % (new_id, quote(attachment['filename']))
new_path = TRAC_ENV + '/files/attachments/ticket/'
new_path += '%s/%s/' % (path_hash[0:3], path_hash)
try:
os.makedirs(new_path)
except:
print
new_path += file_hash
new_path += os.path.splitext(attachment['filename'])[1]
os.rename(old_path, new_path)
except:
errorStr = " * ERROR: couldn't migrate attachment %s" % attachment['filename']
errors.append(errorStr)
print errorStr
print
if TIME_ADJUSTMENT_HACK:
for adjustment in timeAdjustmentHacks:
print adjustment
if len(errors) != 0: