From 5e4a8225335c7f4bd5f3aa62621ac4c9ac5096d6 Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Wed, 17 Dec 2014 12:03:00 +0100 Subject: [PATCH 01/11] fix file reference problem --- .../b2share/modules/b2deposit/b2share_upload_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/b2share_upload_handler.py b/invenio/b2share/modules/b2deposit/b2share_upload_handler.py index b8d306cc64..5aa11ec0a8 100644 --- a/invenio/b2share/modules/b2deposit/b2share_upload_handler.py +++ b/invenio/b2share/modules/b2deposit/b2share_upload_handler.py @@ -92,7 +92,7 @@ def upload(request, sub_id): if (chunks is None) or (int(chunk) == int(chunks) - 1): '''All chunks have been uploaded! Start merging the chunks!''' - merge_chunks_and_create_metadata(upload_dir, name, md5, safename) + filename = merge_chunks_and_create_metadata(upload_dir, name, md5, safename) return filename def merge_chunks_and_create_metadata(upload_dir, name, md5, safename): @@ -118,6 +118,7 @@ def merge_chunks_and_create_metadata(upload_dir, name, md5, safename): metadata_file_path = os.path.join(upload_dir, 'metadata_' + file_unique_name) pickle.dump(file_metadata, open(metadata_file_path, 'wb')) current_app.logger.info("finished uploading: %s, size %d, in %s" % (name, size, file_path)) + return file_unique_name def delete(request, sub_id): """ @@ -170,14 +171,14 @@ def get_file(request, sub_id): "CFG_B2SHARE_UPLOAD_FOLDER") filename = request.args.get('filename') + f = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, sub_id, filename) # make sure that request doesn't go outside the CFG_B2SHARE_UPLOAD_FOLDER if not os.path.samefile( CFG_B2SHARE_UPLOAD_FOLDER, os.path.commonprefix([CFG_B2SHARE_UPLOAD_FOLDER, - os.path.realpath(filename)])): + os.path.realpath(f)])): return "File " + filename + " not found", 404 - f = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, sub_id, filename) if (os.path.isfile(f)): return send_file(f, attachment_filename=filename, as_attachment=True) else: From 999271806e1363bcb34fa230aba208e320763f9f Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Wed, 17 Dec 2014 12:28:53 +0100 Subject: [PATCH 02/11] fix file progress bar --- .../modules/b2deposit/static/js/b2share-deposit.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/static/js/b2share-deposit.js b/invenio/b2share/modules/b2deposit/static/js/b2share-deposit.js index 94717113f0..7f54d21134 100755 --- a/invenio/b2share/modules/b2deposit/static/js/b2share-deposit.js +++ b/invenio/b2share/modules/b2deposit/static/js/b2share-deposit.js @@ -199,7 +199,7 @@ function b2share_init_plupload(selector, url, delete_url, get_file_url) { $.each(uploader.files, function(i, file) { if (file.loaded < file.size){ $("#" + file.id + "_rm").show(); - $('#' + file.id + " .bar").css('width', "0%"); + $('#' + file.id + " .progress-bar").css('width', "0%"); } }); }); @@ -225,7 +225,7 @@ function b2share_init_plupload(selector, url, delete_url, get_file_url) { }); uploader.bind('UploadProgress', function(up, file) { - $('#' + file.id + " .bar").css('width', file.percent + "%"); + $('#' + file.id + " .progress-bar").css('width', file.percent + "%"); setDepositBtnState(); }); @@ -298,7 +298,7 @@ function b2share_init_plupload(selector, url, delete_url, get_file_url) { '' + '' + file.name + '' + '' + plupload.formatSize(file.size) + '' + - '
' + + '
' + '' + ''); $('#filelist #' + file.id).show('fast'); @@ -311,8 +311,8 @@ function b2share_init_plupload(selector, url, delete_url, get_file_url) { }); uploader.bind('FileUploaded', function(up, file, responseObj) { - $('#' + file.id + " .progress").removeClass("progress-striped"); - $('#' + file.id + " .bar").css('width', "100%"); + $('#' + file.id + " .progress-bar").removeClass("active"); + $('#' + file.id + " .progress-bar").css('width', "100%"); $('#' + file.id + '_rm').show(); $('#' + file.id + '_link').html('' + file.name + ''); file.unique_filename = responseObj.response; From 39c31e4787722027709babf21908b8d3301e9dbb Mon Sep 17 00:00:00 2001 From: Sarah Berenji Date: Fri, 19 Dec 2014 09:10:38 +0000 Subject: [PATCH 03/11] fix non-ascii letters and multi-author issues --- invenio/b2share/modules/b2deposit/restful.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/restful.py b/invenio/b2share/modules/b2deposit/restful.py index 340d413696..164e9e0c4c 100644 --- a/invenio/b2share/modules/b2deposit/restful.py +++ b/invenio/b2share/modules/b2deposit/restful.py @@ -91,11 +91,13 @@ def get_value(res, tag): temp = list(flatten(res)) if len(temp) == 6 and tag == "": - return temp[1] + return temp[1].decode('utf-8') elif len(temp) == 8 and tag == "": - return temp[3] + return temp[3].decode('utf-8') elif tag == "file_url": return filter(lambda element: "http" in str(element), temp) + elif tag == "authors": + return [temp[i].decode('utf-8') for i in range(1, len(temp), 6)] return None @@ -119,7 +121,7 @@ def get_record_details(recid): content = [] atts = {} for afile in latest_files: - atts['full_name'] = afile.get_full_name() + atts['full_name'] = afile.get_full_name().decode('utf-8') atts['size'] = afile.get_size() atts['url'] = afile.get_url() content.append(atts.copy()) @@ -141,7 +143,7 @@ def get_record_details(recid): record = create_record(marcxml)[0] authors = record_get_field_instances(record, '100') - file_dict['authors'] = get_value(authors, "") + file_dict['authors'] = get_value(authors, "authors") record_title = record_get_field_instances(record, '245') file_dict['title'] = get_value(record_title, "") From 19e76c40c7eb5ace9ce346110179e84d6b38102a Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Thu, 8 Jan 2015 15:32:10 +0100 Subject: [PATCH 04/11] refactored metadata model file --- .../modules/b2deposit/b2share_model/model.py | 64 ++++++------------- 1 file changed, 18 insertions(+), 46 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/b2share_model/model.py b/invenio/b2share/modules/b2deposit/b2share_model/model.py index 595c3f106d..e261517308 100644 --- a/invenio/b2share/modules/b2deposit/b2share_model/model.py +++ b/invenio/b2share/modules/b2deposit/b2share_model/model.py @@ -180,47 +180,31 @@ def _create_metadata_class(cfg): if not hasattr(cfg, 'fields'): cfg.fields = [] - # TODO: this can be done in a simpler and clearer way now - def basic_field_iter(): + def basic_fields(): + return [f for f in cfg.fields if not ('extra' in f and f['extra'])] - #Normal field if extra is false or not set - for f in cfg.fields: - try: - if not f['extra']: - yield f['name'] - except KeyError: - yield f['name'] - - def optional_field_iter(): - - for f in cfg.fields: - try: - if f['extra']: - yield f['name'] - except KeyError: - pass + def optional_fields(): + return [f for f in cfg.fields if ('extra' in f and f['extra'])] def __init__(self): super(type(self), self).__init__() if len(cfg.fields) > 0: - self.fieldsets.append(FieldSet( - cfg.domain, - basic_fields=list(basic_field_iter()), - optional_fields=list(optional_field_iter()))) + self.fieldsets.append( + FieldSet(cfg.domain, basic_fields=basic_fields(), + optional_fields=optional_fields())) clsname = cfg.domain + "Metadata" args = {'__init__': __init__, '__tablename__': cfg.table_name, '__mapper_args__': {'polymorphic_identity': cfg.table_name}, - 'id': db.Column( - db.Integer, db.ForeignKey('submission_metadata.id'), - primary_key=True), + 'id': db.Column(db.Integer, + db.ForeignKey('submission_metadata.id'), + primary_key=True), 'field_args': {}} #The following function and call just add all external attrs manually def is_external_attr(n): - # don't like this bit; problem is we don't want to include the # db import and I don't know how to exclude them except via name if n in ['db', 'fields']: @@ -236,25 +220,13 @@ def is_external_attr(n): for f in cfg.fields: nullable = not f.get('required', False) args[f['name']] = db.Column(f['col_type'], nullable=nullable) - # Doesn't seem pythonic, but show me a better way - args['field_args'][f['name']] = {} - if 'display_text' in f: - args['field_args'][f['name']]['label'] = f.get('display_text') - if 'description' in f: - args['field_args'][f['name']]['description'] = f.get('description') - if 'data_provide' in f: - args['field_args'][f['name']]['data_provide'] = f.get('data_provide') - if 'data_source' in f: - args['field_args'][f['name']]['data_source'] = f.get('data_source') - if 'default' in f: - args['field_args'][f['name']]['default'] = f.get('default') - if 'placeholder' in f: - args['field_args'][f['name']]['placeholder'] = f.get('placeholder') - if 'value' in f: - args['field_args'][f['name']]['value'] = f.get('value') - if 'other' in f: - args['field_args'][f['name']]['other'] = f.get('other') - if 'cardinality' in f: - args['field_args'][f['name']]['cardinality'] = f.get('cardinality') + field_dict = {} + for k in f: + if k in ['description', 'data_provide', 'data_source', 'default', + 'placeholder', 'value', 'other', 'cardinality']: + field_dict[k] = f.get(k) + elif k == 'display_text': + field_dict['label'] = f.get(k) + args['field_args'][f['name']] = field_dict return type(clsname, (SubmissionMetadata,), args) From 64f6d999c640b7bbd59b2576a1dca01c4357c7f1 Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Fri, 9 Jan 2015 16:10:59 +0100 Subject: [PATCH 05/11] fixed description of the creator field; also indentation --- .../modules/b2deposit/b2share_model/model.py | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/b2share_model/model.py b/invenio/b2share/modules/b2deposit/b2share_model/model.py index e261517308..af4e96ffe9 100644 --- a/invenio/b2share/modules/b2deposit/b2share_model/model.py +++ b/invenio/b2share/modules/b2deposit/b2share_model/model.py @@ -85,16 +85,14 @@ def __init__(self): optional_fields=self.optional_fields))] self.field_args['title'] = { 'placeholder': "Title of the resource", - 'description': - 'The title of the uploaded resource - a name that ' +\ - 'indicates the content to be expected.' + 'description': 'The title of the uploaded resource - a name ' +\ + 'that indicates the content to be expected.' } self.field_args['description'] = { - 'description': - 'A more elaborate description of the resource. ' +\ - 'Focus on a description of ' +\ - 'content making it easy for others to find it and to ' +\ - 'interpret its relevance quickly.' + 'description': 'A more elaborate description of the resource. ' +\ + 'Focus on a description of content making it ' +\ + 'easy for others to find it and to interpret ' +\ + 'its relevance quickly.' } self.field_args['publisher'] = { 'hidden': True, @@ -115,8 +113,7 @@ def __init__(self): } self.field_args['version'] = { 'placeholder': 'e.g. v1.02', - 'description': - 'Denote the version of the resource.' + 'description': 'Denote the version of the resource.' } self.field_args['licence'] = { 'description': 'Specify the license under which this data set '+\ @@ -126,23 +123,21 @@ def __init__(self): } self.field_args['keywords'] = { 'placeholder': "keyword1, keyword2, ...", - 'description': - 'A comma separated list of keywords that ' +\ - 'characterize the content.' + 'description': 'A comma separated list of keywords that ' +\ + 'characterize the content.' } self.field_args['open_access'] = { - 'description': - 'Indicate whether the resource is open or access ' +\ - 'is restricted. In case of restricted access the uploaded files ' +\ - 'will not be public, however the metadata will be.' + 'description': 'Indicate whether the resource is open or ' +\ + 'access is restricted. In case of restricted ' +\ + 'access the uploaded files will not be public, ' +\ + 'however the metadata will be.' } self.field_args['contributors'] = { 'placeholder': 'contributor', 'cardinality': 'n', - 'description': - 'A semicolon separated list of all other ' +\ - 'contributors. Mention all ' +\ - 'other persons that were relevant in the creation of the resource.' + 'description': 'A semicolon separated list of all other ' +\ + 'contributors. Mention all other persons that ' +\ + 'were relevant in the creation of the resource.' } self.field_args['language'] = { 'hidden': True, @@ -154,23 +149,21 @@ def __init__(self): 'data_provide': 'select', 'cardinality': 'n', 'data_source': ['Text', 'Image', 'Video', 'Other'], - 'description': - 'Select the type of the resource.' + 'description': 'Select the type of the resource.' } self.field_args['alternate_identifier'] = { 'placeholder': 'Other reference, such as URI, ISBN, etc.', - 'description': - 'Any kind of other reference such as a URN, URI or an ISBN number.' + 'description': 'Any kind of other reference such as a URN, URI ' +\ + 'or an ISBN number.' } self.field_args['creator'] = { 'placeholder': 'author', 'cardinality': 'n', - 'description': - 'A semicolon separated list of authors of the resource.' + 'description': 'The author(s) of the resource.' } self.field_args['contact_email'] = { 'placeholder': 'contact email', - 'description': 'Contact email information for this record' + 'description': 'Contact email information for this record' } def _create_metadata_class(cfg): From 30b935ac9d60ef65138dd12face9694e214bddc1 Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Sat, 10 Jan 2015 06:11:55 +0100 Subject: [PATCH 06/11] bug fix for previous refactoring --- invenio/b2share/modules/b2deposit/b2share_model/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/b2share_model/model.py b/invenio/b2share/modules/b2deposit/b2share_model/model.py index af4e96ffe9..bc12eaa849 100644 --- a/invenio/b2share/modules/b2deposit/b2share_model/model.py +++ b/invenio/b2share/modules/b2deposit/b2share_model/model.py @@ -174,10 +174,10 @@ def _create_metadata_class(cfg): cfg.fields = [] def basic_fields(): - return [f for f in cfg.fields if not ('extra' in f and f['extra'])] + return [f['name'] for f in cfg.fields if not f.get('extra')] def optional_fields(): - return [f for f in cfg.fields if ('extra' in f and f['extra'])] + return [f['name'] for f in cfg.fields if f.get('extra')] def __init__(self): super(type(self), self).__init__() From 30808ad284e20d03a6eb33ec7e3d1de085fe6ff6 Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Sat, 10 Jan 2015 07:48:59 +0100 Subject: [PATCH 07/11] add domain metadata to REST records + refactoring --- invenio/b2share/modules/b2deposit/restful.py | 233 +++++++++---------- 1 file changed, 106 insertions(+), 127 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/restful.py b/invenio/b2share/modules/b2deposit/restful.py index 164e9e0c4c..67b1db1350 100644 --- a/invenio/b2share/modules/b2deposit/restful.py +++ b/invenio/b2share/modules/b2deposit/restful.py @@ -19,20 +19,18 @@ from __future__ import absolute_import -# from flask import request, current_app +import json + +from flask import current_app, Response from flask.ext.login import current_user -from flask.ext.restful import Resource, abort,\ - fields, marshal -# from flask.ext.restful import reqparse, -# from functools import wraps +from flask.ext.restful import Resource, abort from invenio.ext.restful import require_api_auth, require_header -# from invenio.ext.restful import error_codes - -# from intbitset import intbitset -# from invenio.legacy.dbquery import run_sql -# from invenio.ext.sqlalchemy import db from invenio.modules.editor.models import Bib98x, BibrecBib98x +from invenio.modules.formatter import engine as bibformat_engine + +from invenio.b2share.modules.b2deposit.b2share_model import metadata_classes + MAX_PAGE_SIZE = 20 @@ -44,133 +42,114 @@ require_api_auth(), ] -files_fields = { - 'full_name': fields.String, - 'size': fields.Integer, - 'url': fields.String, - # 'name': fields.String, - # 'comment': fields.String, - # 'description': fields.String, - # 'eformat': fields.String, - # 'magic': fields.String, - # 'status': fields.String, - # 'subformat': fields.String, - # 'superformat': fields.String, - # 'type': fields.String, - # 'version': fields.Integer, -} -output_fields = { - 'recordID': fields.Integer, - 'authors': fields.String, - 'title': fields.String, - 'description': fields.String, - 'domain': fields.String, - 'date': fields.String, - 'pid': fields.String, - 'email': fields.String, - 'file_url': fields.String, - 'licence': fields.String, - 'files': fields.Nested(files_fields), +############################################################################### +# Conversion table for the set of basic fields of a record, +# as expressed in the b2share_model/metdata/metadata.py: +# +# 1. the field name, spelled as in the metadata model +# 2. their encoding as marc fields in the database, +# 3. their multiplicity in the database (True of False) +# +# FIELD_NAME MARC MULTIPLE +basic_fields_meta = { + 'creator': ('100__a', True), + 'title': ('245__a', False), + 'description': ('520__a', False), + 'keywords': ('6531_a', True), + 'contributors': ('700__a', True), + 'domain': ('980__a', False), #\ same marctag + 'resource_type': ('980__a', True), #/ same marctag + 'publication_date': ('260__c', False), + 'contact_email': ('270__m', False), + 'open_access': ('542__l', False), + 'licence': ('540__a', False), + 'version': ('250__a', False), + 'alternate_identifier': ('024__a', False), } -# ========= -# Helpers -# ========= - - -def flatten(lst): - if type(lst) not in (tuple, list): - return (lst,) - if len(lst) == 0: - return tuple(lst) - return flatten(lst[0]) + flatten(lst[1:]) - - -def get_value(res, tag): - temp = list(flatten(res)) - - if len(temp) == 6 and tag == "": - return temp[1].decode('utf-8') - elif len(temp) == 8 and tag == "": - return temp[3].decode('utf-8') - elif tag == "file_url": - return filter(lambda element: "http" in str(element), temp) - elif tag == "authors": - return [temp[i].decode('utf-8') for i in range(1, len(temp), 6)] +def read_basic_medata_field_from_marc(bfo, fieldname): + if fieldname in basic_fields_meta: + marctag = basic_fields_meta[fieldname][0] + multiple = basic_fields_meta[fieldname][1] + if marctag == '980__a': + # duplicated marc tag, serves as domain specifier + # and also as resource_type specifier + ret = bfo.fields(marctag) + if fieldname == 'domain': + ret = [r.lower() for r in ret if r.lower() in metadata_classes()] + else: + ret = [r for r in ret if r.lower() not in metadata_classes()] + return ret if multiple else ", ".join(ret) + elif marctag: + if multiple: + return bfo.fields(marctag) + else: + return bfo.field(marctag) return None +def read_domain_specific_medata_field_from_marc(bfo, fieldname, multiple): + ret = [fx.get('b') for fx in bfo.fields('690__') + if fx.get('a') == fieldname and fx.get('b')] + return ret if multiple else ", ".join(ret) -def get_record_details(recid): - from invenio.legacy.bibdocfile.api import BibRecDocs,\ - InvenioBibDocFileError - from invenio.legacy.bibrecord import create_record,\ - record_get_field_instances - from invenio.legacy.search_engine import print_record +def get_domain_metadata(domain_class, fieldset, bfo): + ret = {} + for fieldname in fieldset.optional_fields + fieldset.basic_fields: + field = domain_class.field_args[fieldname] + multiple = 'cardinality' in field and field['cardinality'] == 'n' + ret[fieldname] = read_domain_specific_medata_field_from_marc(bfo, fieldname, multiple) + return ret +def get_record_details(recid): + from invenio.legacy.bibdocfile.api import BibRecDocs try: recdocs = BibRecDocs(recid) - except InvenioBibDocFileError: + except: + current_app.logger.error("REST API: Error while building BibRecDocs for record %d" % (recid,)) return [] latest_files = recdocs.list_latest_files() if len(latest_files) == 0: + current_app.logger.error("REST API: BibRecDocs reports 0 files for record %d" % (recid,)) return [] - else: - file_dict = {} - content = [] - atts = {} - for afile in latest_files: - atts['full_name'] = afile.get_full_name().decode('utf-8') - atts['size'] = afile.get_size() - atts['url'] = afile.get_url() - content.append(atts.copy()) - # atts['comment'] = afile.get_comment() - # atts['description'] = afile.get_description() - # atts['eformat'] = afile.get_format() - # atts['magic'] = afile.get_magic() - # atts['name'] = afile.get_name() - # atts['status'] = afile.get_status() - # atts['subformat'] = afile.get_subformat() - # atts['superformat'] = afile.get_superformat() - # atts['type'] = afile.get_type() - # atts['version'] = afile.get_version() - - file_dict['files'] = content - file_dict['recordID'] = recid - - marcxml = print_record(recid, 'xm') - record = create_record(marcxml)[0] - - authors = record_get_field_instances(record, '100') - file_dict['authors'] = get_value(authors, "authors") - - record_title = record_get_field_instances(record, '245') - file_dict['title'] = get_value(record_title, "") - - record_description = record_get_field_instances(record, '520') - file_dict['description'] = get_value(record_description, "") - record_domain = record_get_field_instances(record, '980') - file_dict['domain'] = get_value(record_domain, "") - - record_date = record_get_field_instances(record, '260') - file_dict['date'] = get_value(record_date, "") - - record_licence = record_get_field_instances(record, '540') - file_dict['licence'] = get_value(record_licence, "") - - record_PID = record_get_field_instances(record, '024') - file_dict['pid'] = get_value(record_PID, "") - - user_email = record_get_field_instances(record, '856', '0') - file_dict['email'] = get_value(user_email, "") - - file_url = record_get_field_instances(record, '856', '4') - file_dict['file_url'] = get_value(file_url, "file_url") - - return file_dict + # bibformat uses get_record, usually is one db hit per object; should be fastest + bfo = bibformat_engine.BibFormatObject(recid) + + # first put the recordID and list of files + ret = { + 'recordID': recid, + 'files': [{ + 'name': afile.get_full_name().decode('utf-8'), + 'size': afile.get_size(), + 'url': afile.get_full_url(), + } for afile in latest_files ], + } + + # add basic metadata fields + for fieldname in basic_fields_meta: + ret[fieldname] = read_basic_medata_field_from_marc(bfo, fieldname) + + # add 'PID' + for fx in bfo.fields('0247_'): + if fx.get('2') == "PID": + ret[fx.get('2')] = fx.get('a') + + # add 'domain' + domain = read_basic_medata_field_from_marc(bfo, 'domain') + ret['domain'] = domain + + # add domain-specific metadata fields + if domain not in metadata_classes(): + current_app.logger.error("Bad domain metadata class for record %d" % (recid,)) + else: + domain_class = metadata_classes()[domain]() + for fieldset in domain_class.fieldsets: + if fieldset.name != 'Generic': + ret['domain_metadata'] = get_domain_metadata(domain_class, fieldset, bfo) + return ret # ========= # Resources @@ -206,11 +185,11 @@ def get(self, oauth, domain_name, **kwargs): # get domain id from domain name domain = Bib98x.query.filter_by(value=domain_name).first() if domain is None: - abort(404, message="Please try a valid domain name:\ - Generic, EUON, DRIHM, Linguistics, BBMRI", status=404) + abort(404, status=404, + message="Please try a valid domain name: " + + ", ".join(metadata_classes().keys())) domain_records = BibrecBib98x.query.filter_by(id_bibxxx=domain.id).all() - record_ids = [] record_ids = [record.id_bibrec for record in domain_records] record_list = [] @@ -218,7 +197,7 @@ def get(self, oauth, domain_name, **kwargs): record_details = get_record_details(record_id) record_list.append(record_details) - return marshal(record_list, output_fields) + return record_list @require_header('Content-Type', 'application/json') def post(self, oauth): @@ -281,7 +260,7 @@ def get(self, oauth, **kwargs): for record_id in record_ids[page_offset * page_size:page_offset * page_size + page_size]: record_details = get_record_details(record_id) record_list.append(record_details) - return marshal(record_list, output_fields) + return record_list def post(self, oauth): """ @@ -319,7 +298,7 @@ def get(self, oauth, record_id): record_details = get_record_details(record_id) if not record_details: abort(404, message="Deposition not found", status=404) - return marshal(record_details, output_fields) + return record_details def post(self, oauth): """ From 6821300f50b4747c21105f86948c2177d44bd90b Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Sat, 10 Jan 2015 08:20:53 +0100 Subject: [PATCH 08/11] using a parser for page_size and page_offset --- invenio/b2share/modules/b2deposit/restful.py | 75 ++++++-------------- 1 file changed, 23 insertions(+), 52 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/restful.py b/invenio/b2share/modules/b2deposit/restful.py index 67b1db1350..458b22d289 100644 --- a/invenio/b2share/modules/b2deposit/restful.py +++ b/invenio/b2share/modules/b2deposit/restful.py @@ -19,11 +19,10 @@ from __future__ import absolute_import -import json - -from flask import current_app, Response +from flask import current_app from flask.ext.login import current_user -from flask.ext.restful import Resource, abort +from flask.ext.restful import Resource, abort, reqparse + from invenio.ext.restful import require_api_auth, require_header from invenio.modules.editor.models import Bib98x, BibrecBib98x @@ -32,7 +31,8 @@ from invenio.b2share.modules.b2deposit.b2share_model import metadata_classes -MAX_PAGE_SIZE = 20 +PAGE_SIZE = 20 +MAX_PAGE_SIZE = 100 # ========= @@ -154,6 +154,11 @@ def get_record_details(recid): # ========= # Resources # ========= + +pager = reqparse.RequestParser() +pager.add_argument('page_size', type=int, help='Number of items to return') +pager.add_argument('page_offset', type=int, help='Index of the first returned item') + class DepositionDomains(Resource): """ Collection of depositions: specific Domain @@ -165,22 +170,11 @@ def get(self, oauth, domain_name, **kwargs): List all deposits for a specific domain """ - - if len(kwargs) == 0: - page_size = 5 - page_offset = 0 - elif len(kwargs) == 1: - if kwargs['page_size'] > MAX_PAGE_SIZE: - page_size = MAX_PAGE_SIZE - else: - page_size = kwargs['page_size'] - page_offset = 0 - elif len(kwargs) == 2: - if kwargs['page_size'] > MAX_PAGE_SIZE: - page_size = MAX_PAGE_SIZE - else: - page_size = kwargs['page_size'] - page_offset = kwargs['page_offset'] + pag = pager.parse_args() + page_size = pag.get('page_size', PAGE_SIZE) + page_offset = pag.get('page_offset', 0) + if page_size > MAX_PAGE_SIZE: + page_size = MAX_PAGE_SIZE # get domain id from domain name domain = Bib98x.query.filter_by(value=domain_name).first() @@ -236,21 +230,11 @@ def get(self, oauth, **kwargs): from invenio.legacy.search_engine import perform_request_search # from invenio.modules.accounts.models import User - if len(kwargs) == 0: - page_size = 5 - page_offset = 0 - elif len(kwargs) == 1: - if kwargs['page_size'] > MAX_PAGE_SIZE: - page_size = MAX_PAGE_SIZE - else: - page_size = kwargs['page_size'] - page_offset = 0 - elif len(kwargs) == 2: - if kwargs['page_size'] > MAX_PAGE_SIZE: - page_size = MAX_PAGE_SIZE - else: - page_size = kwargs['page_size'] - page_offset = kwargs['page_offset'] + pag = pager.parse_args() + page_size = pag.get('page_size') or PAGE_SIZE + page_offset = pag.get('page_offset') or 0 + if page_size > MAX_PAGE_SIZE: + page_size = MAX_PAGE_SIZE email_field = "8560_" email = current_user['email'] @@ -328,19 +312,6 @@ def patch(self, oauth): def setup_app(app, api): - api.add_resource( - DepositionDomains, - '/api/depositions/', - '/api/depositions//', - '/api/depositions///', - ) - api.add_resource( - DepositionListRecord, - '/api/depositions/', - '/api/depositions//', - '/api/depositions//', - ) - api.add_resource( - DepositionRecord, - '/api/deposit/', - ) + api.add_resource(DepositionDomains, '/api/depositions/') + api.add_resource(DepositionListRecord, '/api/depositions/') + api.add_resource(DepositionRecord, '/api/deposit/') From e2336fe1dae89f3fe4612d8ba40525d35ab75a76 Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Sun, 11 Jan 2015 18:22:05 +0100 Subject: [PATCH 09/11] REST API depositing: able to upload files; work in progress --- .../modules/b2deposit/b2share_marc_handler.py | 29 ++- .../b2deposit/b2share_upload_handler.py | 2 + invenio/b2share/modules/b2deposit/restful.py | 227 +++++++++--------- 3 files changed, 139 insertions(+), 119 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/b2share_marc_handler.py b/invenio/b2share/modules/b2deposit/b2share_marc_handler.py index 7e82a0ff1b..5c9eda0707 100644 --- a/invenio/b2share/modules/b2deposit/b2share_marc_handler.py +++ b/invenio/b2share/modules/b2deposit/b2share_marc_handler.py @@ -112,19 +112,11 @@ def create_recid(): return run_sql("INSERT INTO bibrec(creation_date, modification_date) " "values(NOW(), NOW())") - -def add_file_info(rec, form, email, sub_id, recid): - """ - Adds the path to the file and access rights to ther record. - """ +def get_depositing_files_metadata(deposit_id): CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get("CFG_B2SHARE_UPLOAD_FOLDER") - upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, sub_id) + upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) files = os.listdir(upload_dir) - if 'open_access' in form: - fft_status = 'firerole: allow any\n' - else: - fft_status = 'firerole: allow email "{0}"\ndeny all'.format( - email) + ret = [] for f in files: path = os.path.join(upload_dir, f) if f.startswith('metadata_'): @@ -139,7 +131,22 @@ def add_file_info(rec, form, email, sub_id, recid): else: current_app.logger.error('Submitted file \'%s\' is missing metadata file, using default' % f) metadata = dict(name=f, file=path, size=str(os.path.getsize(path))) + ret.append(metadata) + return ret + +def add_file_info(rec, form, email, sub_id, recid): + """ + Adds the path to the file and access rights to ther record. + """ + if 'open_access' in form: + fft_status = 'firerole: allow any\n' + else: + fft_status = 'firerole: allow email "{0}"\ndeny all'.format( + email) + for metadata in get_depositing_files_metadata(sub_id): + f = metadata.name + path = metadata.file record_add_field(rec, 'FFT', subfields=[('a', path), ('n', metadata['name']), # name of the file diff --git a/invenio/b2share/modules/b2deposit/b2share_upload_handler.py b/invenio/b2share/modules/b2deposit/b2share_upload_handler.py index 5aa11ec0a8..2336d9cc69 100644 --- a/invenio/b2share/modules/b2deposit/b2share_upload_handler.py +++ b/invenio/b2share/modules/b2deposit/b2share_upload_handler.py @@ -110,7 +110,9 @@ def merge_chunks_and_create_metadata(upload_dir, name, md5, safename): with open(chunk, 'rb') as chunk_fd: shutil.copyfileobj(chunk_fd, destination) os.remove(chunk) + create_file_metadata(upload_dir, name, file_unique_name, file_path) +def create_file_metadata(upload_dir, name, file_unique_name, file_path): size = os.path.getsize(file_path) file_metadata = dict(name=name, file=file_path, size=size) diff --git a/invenio/b2share/modules/b2deposit/restful.py b/invenio/b2share/modules/b2deposit/restful.py index 458b22d289..8b44fc3b03 100644 --- a/invenio/b2share/modules/b2deposit/restful.py +++ b/invenio/b2share/modules/b2deposit/restful.py @@ -19,30 +19,25 @@ from __future__ import absolute_import -from flask import current_app -from flask.ext.login import current_user -from flask.ext.restful import Resource, abort, reqparse +import os, os.path, uuid +from flask import current_app, request +from flask.ext.restful import Resource, abort, reqparse -from invenio.ext.restful import require_api_auth, require_header +from invenio.ext.restful import require_api_auth from invenio.modules.editor.models import Bib98x, BibrecBib98x from invenio.modules.formatter import engine as bibformat_engine from invenio.b2share.modules.b2deposit.b2share_model import metadata_classes - +from invenio.b2share.modules.b2deposit.b2share_upload_handler \ + import encode_filename, get_extension, create_file_metadata +from invenio.b2share.modules.b2deposit.b2share_marc_handler \ + import get_depositing_files_metadata PAGE_SIZE = 20 MAX_PAGE_SIZE = 100 -# ========= -# Mix-ins -# ========= -deposit_decorators = [ - require_api_auth(), -] - - ############################################################################### # Conversion table for the set of basic fields of a record, # as expressed in the b2share_model/metdata/metadata.py: @@ -159,20 +154,27 @@ def get_record_details(recid): pager.add_argument('page_size', type=int, help='Number of items to return') pager.add_argument('page_offset', type=int, help='Index of the first returned item') -class DepositionDomains(Resource): - """ - Collection of depositions: specific Domain +class B2Resource(Resource): + method_decorators = [require_api_auth()] + def get(self, oauth, **kwargs): abort(405) + def post(self, oauth, **kwargs): abort(405) + def put(self, oauth, **kwargs): abort(405) + def delete(self, oauth, **kwargs): abort(405) + def head(self, oauth, **kwargs): abort(405) + def options(self, oauth, **kwargs): abort(405) + def patch(self, oauth, **kwargs): abort(405) + +class ListRecordsByDomain(B2Resource): """ - method_decorators = deposit_decorators + The resource representing the collection of all records, filtered by domain + Can only list the available resources, + use POST to /depositions to create a new one + """ def get(self, oauth, domain_name, **kwargs): - """ - List all deposits for a specific domain - - """ pag = pager.parse_args() - page_size = pag.get('page_size', PAGE_SIZE) - page_offset = pag.get('page_offset', 0) + page_size = pag.get('page_size') or PAGE_SIZE + page_offset = pag.get('page_offset') or 0 if page_size > MAX_PAGE_SIZE: page_size = MAX_PAGE_SIZE @@ -187,46 +189,22 @@ def get(self, oauth, domain_name, **kwargs): record_ids = [record.id_bibrec for record in domain_records] record_list = [] - for record_id in record_ids[page_offset * page_size:page_offset * page_size + page_size]: + start = page_offset * page_size + for record_id in record_ids[start : start + page_size]: record_details = get_record_details(record_id) record_list.append(record_details) return record_list - @require_header('Content-Type', 'application/json') - def post(self, oauth): - """ - Create a new deposition - """ - abort(405) - - def put(self, oauth): - abort(405) - - def delete(self, oauth): - abort(405) - - def head(self, oauth): - abort(405) - - def options(self, oauth): - abort(405) - - def patch(self, oauth): - abort(405) - -class DepositionListRecord(Resource): - """ - Collection of depositions: specific user +class ListRecords(B2Resource): """ - method_decorators = deposit_decorators + The resource representing the collection of all records + Can only list the available records, + use POST to /depositions to create a new one + """ def get(self, oauth, **kwargs): - """ - List depositions - - """ from invenio.legacy.search_engine import perform_request_search # from invenio.modules.accounts.models import User @@ -236,82 +214,115 @@ def get(self, oauth, **kwargs): if page_size > MAX_PAGE_SIZE: page_size = MAX_PAGE_SIZE - email_field = "8560_" - email = current_user['email'] - record_ids = perform_request_search(f=email_field, p=email, of="id") - record_list = [] + # enumerate all valid ids + record_ids = perform_request_search(of="id", sf="005") - for record_id in record_ids[page_offset * page_size:page_offset * page_size + page_size]: + record_list = [] + start = page_offset * page_size + for record_id in record_ids[start : start + page_size]: record_details = get_record_details(record_id) record_list.append(record_details) return record_list - def post(self, oauth): - """ - Create a new deposition - """ - pass - def put(self, oauth): - abort(405) - - def delete(self, oauth): - abort(405) - - def head(self, oauth): - abort(405) - - def options(self, oauth): - abort(405) - - def patch(self, oauth): - abort(405) - - -class DepositionRecord(Resource): +class RecordRes(B2Resource): """ - Deposition item + A record resource is (for now) immutable, can be read with GET """ - method_decorators = deposit_decorators - - def get(self, oauth, record_id): - """ - Retrieve requested record - - """ + def get(self, oauth, record_id, **kwargs): record_details = get_record_details(record_id) if not record_details: abort(404, message="Deposition not found", status=404) return record_details - def post(self, oauth): - """ - Create a new deposition - """ - pass - def put(self, oauth): - abort(405) - - def delete(self, oauth): - abort(405) - - def head(self, oauth): +class ListDepositions(B2Resource): + """ + The resource representing the active deposits. + A deposit is mutable and private, while a record is immutable and public + """ + def get(self, oauth, **kwargs): + # TODO: ideally we'd be able to list all depositIDs of the current user + # but right now we don't know which ones belong to the user abort(405) - def options(self, oauth): - abort(405) + def post(self, oauth, **kwargs): + """Creates a new deposition""" + CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( + "CFG_B2SHARE_UPLOAD_FOLDER") + deposit_id = uuid.uuid1().hex + upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) + os.makedirs(upload_dir) + location = "/api/deposition/" + deposit_id, + json_data = { + 'message': "New deposition created", + 'location': "/api/deposition/" + deposit_id, + } + return json_data, 201, {'Location' : location} # return location header + + +class DepositionFiles(B2Resource): + """ + The resource representing the list of files in a deposition. + """ + def check_user(self, deposit_id): + # TODO: make sure the deposition folder is only readable by its owner + pass - def patch(self, oauth): + def get(self, oauth, deposit_id, **kwargs): + """Get the list of files already uploaded""" + CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( + "CFG_B2SHARE_UPLOAD_FOLDER") + upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) + if not os.path.exists(upload_dir): + # don't use abort(404), it adds its own bad error message + return {'message':'bad deposit_id parameter', 'status':404}, 404 + self.check_user(deposit_id) + files = [{'name': f['name'], 'size': f['size'] } + for f in get_depositing_files_metadata(deposit_id)] + return files + + def post(self, oauth, deposit_id, **kwargs): + """Adds a new deposition file""" + CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( + "CFG_B2SHARE_UPLOAD_FOLDER") + upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) + if not os.path.exists(upload_dir): + # don't use abort(404), it adds its own bad error message + return {'message':'bad deposit_id parameter', 'status':404}, 404 + + self.check_user(deposit_id) + + file_names = [] + for f in request.files.values(): + safename, md5 = encode_filename(f.filename) + file_unique_name = safename + "_" + md5 + get_extension(safename) + file_path = os.path.join(upload_dir, file_unique_name) + f.save(file_path) + create_file_metadata(upload_dir, f.filename, file_unique_name, file_path) + file_names.append({'name':f.filename}) + return {'message':'File(s) saved', 'files':file_names} + +class DepositionCommit(B2Resource): + """ + The resource representing a deposition commit action. + By POSTing valid metadata here the deposition is transformed into a record + """ + def post(self, oauth, deposit_id, **kwargs): + """Creates a new deposition""" + # import ipdb; ipdb.set_trace() abort(405) - # # Register API resources # def setup_app(app, api): - api.add_resource(DepositionDomains, '/api/depositions/') - api.add_resource(DepositionListRecord, '/api/depositions/') - api.add_resource(DepositionRecord, '/api/deposit/') + api.add_resource(ListRecords, '/api/records/') + api.add_resource(ListRecordsByDomain, '/api/records/') + api.add_resource(RecordRes, '/api/record/') + + api.add_resource(ListDepositions, '/api/depositions/') + api.add_resource(DepositionFiles, '/api/deposition/') + # api.add_resource(DepositionCommit, '/api/deposition//commit') From 1ddd2ab0d317805b1acdc1eb1718de46fcd3367f Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Sun, 11 Jan 2015 23:37:13 +0100 Subject: [PATCH 10/11] fixed #499: REST API is now functional for both reading and depositing --- .../b2deposit/b2share_deposit_handler.py | 4 +- .../modules/b2deposit/b2share_marc_handler.py | 29 ++-- invenio/b2share/modules/b2deposit/restful.py | 155 ++++++++++++++++-- 3 files changed, 159 insertions(+), 29 deletions(-) diff --git a/invenio/b2share/modules/b2deposit/b2share_deposit_handler.py b/invenio/b2share/modules/b2deposit/b2share_deposit_handler.py index 5a36047ba5..37db61c9ee 100644 --- a/invenio/b2share/modules/b2deposit/b2share_deposit_handler.py +++ b/invenio/b2share/modules/b2deposit/b2share_deposit_handler.py @@ -31,7 +31,7 @@ from b2share_model.HTML5ModelConverter import HTML5ModelConverter from b2share_model import metadata_classes -import b2share_marc_handler as mh +import b2share_marc_handler @@ -104,7 +104,7 @@ def addmeta(request, sub_id): meta_form = MetaForm(request.form, meta) if meta_form.validate_on_submit(): - recid, marc = mh.create_marc( + recid, marc = b2share_marc_handler.create_marc( request.form, sub_id, current_user['email']) tmp_file = write_marc_to_temp_file(marc) # all usual tasks have priority 0; we want the bibuploads to run first diff --git a/invenio/b2share/modules/b2deposit/b2share_marc_handler.py b/invenio/b2share/modules/b2deposit/b2share_marc_handler.py index 5c9eda0707..1205724d2b 100644 --- a/invenio/b2share/modules/b2deposit/b2share_marc_handler.py +++ b/invenio/b2share/modules/b2deposit/b2share_marc_handler.py @@ -37,24 +37,26 @@ def add_basic_fields(rec, form, email): """ # why aren't subfields a dictionary?! try: - if form['title']: + if form.get('title'): record_add_field(rec, '245', subfields=[('a', remove_html_markup(form['title']))]) - if form['creator']: + if form.get('creator'): fields = form.getlist('creator') for f in fields: if f and not f.isspace(): record_add_field(rec, '100', subfields=[('a', remove_html_markup(f.strip()))]) - if form['domain']: + if form.get('domain'): record_add_field(rec, '980', subfields=[('a', remove_html_markup(form['domain']))]) + pubfields = [] - if form['publisher']: + if form.get('publisher'): pubfields.append(('b', remove_html_markup(form['publisher']))) if form.get('publication_date'): pubfields.append(('c', remove_html_markup(form['publication_date']))) if pubfields: record_add_field(rec, '260', subfields=pubfields) + record_add_field(rec, '856', ind1='0', subfields=[('f', email)]) if 'open_access' in form: @@ -62,21 +64,21 @@ def add_basic_fields(rec, form, email): else: record_add_field(rec, '542', subfields=[('l', 'restricted')]) - if form['licence']: + if form.get('licence'): record_add_field(rec, '540', subfields=[('a', remove_html_markup(form['licence']))]) record_add_field(rec, '520', subfields=[('a', remove_html_markup(form['description']))]) - if form['contact_email']: + if form.get('contact_email'): record_add_field(rec,'270',subfields=[('m', remove_html_markup(form['contact_email']))]) - if form['keywords']: + if form.get('keywords'): for kw in form['keywords'].split(','): if kw and not kw.isspace(): record_add_field(rec, '653', ind1='1', subfields=[('a', remove_html_markup(kw.strip()))]) - if 'contributors' in form and form['contributors']: + if form.get('contributors'): fields = form.getlist('contributors') for f in fields: if f and not f.isspace(): @@ -85,16 +87,16 @@ def add_basic_fields(rec, form, email): record_add_field(rec, '546', subfields=[('a', remove_html_markup(form['language']))]) # copying zenodo here, but I don't think 980 is the right MARC field - if 'resource_type' in form and form['resource_type']: + if form.get('resource_type'): fields = form.getlist('resource_type') for f in fields: record_add_field(rec, '980', subfields=[('a', remove_html_markup(form['resource_type']))]) - if 'alternate_identifier' in form and form['alternate_identifier']: + if form.get('alternate_identifier'): record_add_field(rec, '024', subfields=[('a', remove_html_markup(form['alternate_identifier']))]) - if 'version' in form and form['version']: + if form.get('version'): record_add_field(rec, '250', subfields=[('a', remove_html_markup(form['version']))]) CFG_SITE_NAME = current_app.config.get("CFG_SITE_NAME") @@ -145,8 +147,7 @@ def add_file_info(rec, form, email, sub_id, recid): fft_status = 'firerole: allow email "{0}"\ndeny all'.format( email) for metadata in get_depositing_files_metadata(sub_id): - f = metadata.name - path = metadata.file + path = metadata['file'] record_add_field(rec, 'FFT', subfields=[('a', path), ('n', metadata['name']), # name of the file @@ -160,7 +161,7 @@ def add_file_info(rec, form, email, sub_id, recid): #seems to be impossible to add file size data, thought this would work CFG_SITE_SECURE_URL = current_app.config.get("CFG_SITE_SECURE_URL") - url = "{0}/record/{1}/files/{2}".format(CFG_SITE_SECURE_URL, recid, f) + url = "{0}/record/{1}/files/{2}".format(CFG_SITE_SECURE_URL, recid, metadata['name']) record_add_field(rec, '856', ind1='4', subfields=[('u', url), ('s', str(os.path.getsize(path))), diff --git a/invenio/b2share/modules/b2deposit/restful.py b/invenio/b2share/modules/b2deposit/restful.py index 8b44fc3b03..5bdf493e2b 100644 --- a/invenio/b2share/modules/b2deposit/restful.py +++ b/invenio/b2share/modules/b2deposit/restful.py @@ -22,17 +22,27 @@ import os, os.path, uuid from flask import current_app, request +from flask.ext.login import current_user from flask.ext.restful import Resource, abort, reqparse +from werkzeug.datastructures import ImmutableMultiDict + +from wtforms.ext.sqlalchemy.orm import model_form + from invenio.ext.restful import require_api_auth from invenio.modules.editor.models import Bib98x, BibrecBib98x from invenio.modules.formatter import engine as bibformat_engine from invenio.b2share.modules.b2deposit.b2share_model import metadata_classes +from invenio.b2share.modules.b2deposit.b2share_model.HTML5ModelConverter \ + import HTML5ModelConverter from invenio.b2share.modules.b2deposit.b2share_upload_handler \ import encode_filename, get_extension, create_file_metadata from invenio.b2share.modules.b2deposit.b2share_marc_handler \ - import get_depositing_files_metadata + import get_depositing_files_metadata, create_marc +from invenio.b2share.modules.b2deposit.b2share_deposit_handler \ + import write_marc_to_temp_file, FormWithKey + PAGE_SIZE = 20 MAX_PAGE_SIZE = 100 @@ -190,10 +200,12 @@ def get(self, oauth, domain_name, **kwargs): record_list = [] start = page_offset * page_size - for record_id in record_ids[start : start + page_size]: + stop = start + page_size + for record_id in record_ids[start : stop]: record_details = get_record_details(record_id) record_list.append(record_details) - + if stop < len(record_ids): + record_list.append('...') # continuation indicator return record_list @@ -219,9 +231,12 @@ def get(self, oauth, **kwargs): record_list = [] start = page_offset * page_size - for record_id in record_ids[start : start + page_size]: + stop = start + page_size + for record_id in record_ids[start : stop]: record_details = get_record_details(record_id) record_list.append(record_details) + if stop < len(record_ids): + record_list.append('...') # continuation indicator return record_list @@ -243,11 +258,17 @@ class ListDepositions(B2Resource): """ def get(self, oauth, **kwargs): # TODO: ideally we'd be able to list all depositIDs of the current user - # but right now we don't know which ones belong to the user + # but right now we don't know which ones belong to this user abort(405) def post(self, oauth, **kwargs): - """Creates a new deposition""" + """ + Creates a new deposition + + Test this with: + $ curl -v -X POST http://0.0.0.0:4000/api/depositions?access_token=xxx + should return a new Location URL + """ CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( "CFG_B2SHARE_UPLOAD_FOLDER") deposit_id = uuid.uuid1().hex @@ -261,6 +282,19 @@ def post(self, oauth, **kwargs): return json_data, 201, {'Location' : location} # return location header +class Deposition(B2Resource): + """ + The resource representing a deposition. + """ + def get(self, oauth, deposit_id, **kwargs): + """ + Test this with: + $ curl -v http://0.0.0.0:4000/api/deposition/DEPOSITION_ID?access_token=xxx + """ + prefix = '/api/deposition/'+deposit_id + return {'message':'valid deposition resource', + 'locations': [ prefix + '/files', prefix + '/commit']} + class DepositionFiles(B2Resource): """ The resource representing the list of files in a deposition. @@ -270,7 +304,13 @@ def check_user(self, deposit_id): pass def get(self, oauth, deposit_id, **kwargs): - """Get the list of files already uploaded""" + """ + Get the list of files already uploaded + + Test this with: + $ curl -v http://0.0.0.0:4000/api/deposition/DEPOSITION_ID/files?access_token=xxx + should return the list of deposited files + """ CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( "CFG_B2SHARE_UPLOAD_FOLDER") upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) @@ -283,7 +323,14 @@ def get(self, oauth, deposit_id, **kwargs): return files def post(self, oauth, deposit_id, **kwargs): - """Adds a new deposition file""" + """ + Adds a new deposition file + + Test this with: + $ curl -v -X POST -F file=@TestFileToBeUploaded.txt + http://0.0.0.0:4000/api/deposition/DEPOSITION_ID/files?access_token=xxx + should return the newly deposited files + """ CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( "CFG_B2SHARE_UPLOAD_FOLDER") upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) @@ -309,9 +356,90 @@ class DepositionCommit(B2Resource): By POSTing valid metadata here the deposition is transformed into a record """ def post(self, oauth, deposit_id, **kwargs): - """Creates a new deposition""" - # import ipdb; ipdb.set_trace() - abort(405) + """ + Creates a new deposition + + Test this with: + $ curl -v -X POST -H "Content-Type: application/json" + -d '{"domain":"generic", "title":"REST Test Title", "description":"REST Test Description"}' + http://0.0.0.0:4000/api/deposition/DEPOSITION_ID/commit\?access_token\=xxx + """ + CFG_B2SHARE_UPLOAD_FOLDER = current_app.config.get( + "CFG_B2SHARE_UPLOAD_FOLDER") + upload_dir = os.path.join(CFG_B2SHARE_UPLOAD_FOLDER, deposit_id) + if not os.path.exists(upload_dir): + # don't use abort(404), it adds its own bad error message + return {'message':'bad deposit_id parameter', 'status':404}, 404 + if not os.listdir(upload_dir): + return {'message':'no files: add files to this deposition first', 'status':400}, 400 + + try: + form = request.get_json() + except: + return {'message':'Invalid POST data', 'status':400}, 400 + + domain = form.get('domain', '').lower() + if domain in metadata_classes(): + meta = metadata_classes()[domain]() + else: + domains = ", ".join(metadata_classes().keys()) + json_data = { + 'message': 'Invalid domain. The submitted metadata must '+\ + 'contain a valid "domain" field. Valid domains '+\ + 'are: '+ domains, + 'status': 400, + } + return json_data, 400 + + if 'open_access' not in form: + return {'message':'open_access boolean field required', 'status':400}, 400 + if not form['open_access'] or form['open_access'] == 'restricted': + del form['open_access'] # action required by the b2share_marc_handler + + if not form.get('language'): + form['language'] = meta.language_default + + form = ImmutableMultiDict(form) + + MetaForm = model_form(meta.__class__, base_class=FormWithKey, + exclude=['submission', 'submission_type'], + field_args=meta.field_args, + converter=HTML5ModelConverter()) + + meta_form = MetaForm(form, meta, csrf_enabled=False) + + if meta_form.validate_on_submit(): + recid, marc = create_marc(form, deposit_id, current_user['email']) + tmp_file = write_marc_to_temp_file(marc) + # all usual tasks have priority 0; we want the bibuploads to run first + from invenio.legacy.bibsched.bibtask import task_low_level_submission + task_low_level_submission('bibupload', 'webdeposit', '--priority', '1', '-r', tmp_file) + + #TODO: remove the existing deposition folder?; the user can now + # repeatedly create records with the same deposition + + location = "/api/record/%d" % (recid,) + json_data = { + 'message': "New record submitted for processing", + 'location': "/api/record/%d" % (recid,) + } + return json_data, 201, {'Location':location} # return location header + else: + fields = {} + for (fname, field) in meta.field_args.iteritems(): + if not field.get('hidden'): + fields[fname] = { 'description' : field.get('description') } + if field.get('cardinality') == 'n': + fields[fname]['multiple'] = True + if field.get('data_source'): + fields[fname]['options'] = field.get('data_source') + + json_data = { + 'message': 'Invalid metadata, please review the required fields', + 'status': 400, + 'fields': fields, + } + return json_data, 400 # # Register API resources @@ -324,5 +452,6 @@ def setup_app(app, api): api.add_resource(RecordRes, '/api/record/') api.add_resource(ListDepositions, '/api/depositions/') - api.add_resource(DepositionFiles, '/api/deposition/') - # api.add_resource(DepositionCommit, '/api/deposition//commit') + api.add_resource(Deposition, '/api/deposition/') + api.add_resource(DepositionFiles, '/api/deposition//files') + api.add_resource(DepositionCommit, '/api/deposition//commit') From 4434d486ff7cc96511868a0af366a4c7b5c17185 Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Mon, 12 Jan 2015 05:30:22 +0100 Subject: [PATCH 11/11] improved 'required' specification of metadata fields --- invenio/b2share/modules/b2deposit/restful.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/invenio/b2share/modules/b2deposit/restful.py b/invenio/b2share/modules/b2deposit/restful.py index 5bdf493e2b..da85471194 100644 --- a/invenio/b2share/modules/b2deposit/restful.py +++ b/invenio/b2share/modules/b2deposit/restful.py @@ -380,7 +380,8 @@ def post(self, oauth, deposit_id, **kwargs): domain = form.get('domain', '').lower() if domain in metadata_classes(): - meta = metadata_classes()[domain]() + metaclass = metadata_classes()[domain] + meta = metaclass() else: domains = ", ".join(metadata_classes().keys()) json_data = { @@ -429,6 +430,8 @@ def post(self, oauth, deposit_id, **kwargs): for (fname, field) in meta.field_args.iteritems(): if not field.get('hidden'): fields[fname] = { 'description' : field.get('description') } + if self.is_required_field(metaclass, fname): + fields[fname]['required'] = True if field.get('cardinality') == 'n': fields[fname]['multiple'] = True if field.get('data_source'): @@ -441,6 +444,16 @@ def post(self, oauth, deposit_id, **kwargs): } return json_data, 400 + def is_required_field(self, cls, propname): + from sqlalchemy.orm import class_mapper + import sqlalchemy + for prop in class_mapper(cls).iterate_properties: + if isinstance(prop, sqlalchemy.orm.ColumnProperty) and prop.key == propname: + for col in prop.columns: + if col.nullable == False: + return True + return False + # # Register API resources #