hadrians-wall.py

File contents

#!/usr/bin/python
# -*- coding: utf-8 -*-

from csv import DictReader
import datetime
import logging
from optparse import OptionParser
import re
import sys

from DateTime import DateTime
from Products.CMFCore.utils import getToolByName

from pleiades.bulkup import secure, setup_cmfuid

root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s"))
root_logger.addHandler(handler)
LOG = logging.getLogger('pleiades.scripts')

MESSAGE = "Milecastles and Turrets from Scott Vanderbilt (sarcannon) in January 2012"

def main(context, reader):
    
    catalog = getToolByName(context, 'portal_catalog')
    repo = getToolByName(context, 'portal_repository')
    wftool = getToolByName(context, 'portal_workflow')
    utils = getToolByName(context, 'plone_utils')

    import transaction
    savepoint = transaction.savepoint()
    try:

        # First we will sort locations by their parent place.
        records = {}
        for row in reader:
            
            ptype = row['type']
            pid = str(row['pid'])

            if ptype == 'Place' and pid not in records:
                records[pid] = {'row': row.copy(), 'locations': []}
            elif ptype == 'Location':
                records[pid]['locations'].append(row.copy())

        # Next, create content from this mapping.
        places = context['places']
        for record in records.values():
            prow = record['row']
            locations = record['locations']
            
            LOG.info(
                "Place: %s, locations: %s", 
                record['row']['title'], list(r['title'] for r in locations) )
            
            # Create the place.
            pid = places.invokeFactory(
                    'Place',
                    places.generateId(prefix=''),
                    title=prow['title'],
                    description=prow['description'],
                    placeType=['fort'],
                    modernLocation=None,
                    creators=['sarcanon'],
                    contributors=['sgillies', 'thomase'],
                    originalProvenance='Pleiades'
                    )
            
            place = places[pid]
            
            LOG.info(
                "Created place: %s, %s",
                place.Title(), place.Description() )

            place.addReference(places['91358'], 'connectsWith')

            # Determine whether a Milecastle or Turret and add standard
            # citations accordingly.
            mcn = None
            m = re.search(r"(Milecastle\s+(\d+))", place.Title())
            if m:
                mcn = m.group(2)
            if not mcn:
                m = re.search(r"(Turret\s+(\d+)[A|B])", place.Title())
                if m:
                    mcn = m.group(2)
            
            citations = [
                dict(
                    identifier="http://wikipedia.org/wiki/Hadrian%27s_Wall", 
                    range="Wikipedia, Hadrian's Wall", 
                    type="seeAlso") ]

            if mcn is not None:
                citations.extend([
                    dict(
                        identifier="http://wikipedia.org/wiki/Milecastle", 
                        range="Wikipedia, Milecastle", 
                        type="seeAlso"),
                    dict(
                        identifier="http://wikipedia.org/wiki/Milecastle_%s" % mcn, 
                        range="Wikipedia, Milecastle %s" % mcn, 
                        type="seeFurther")
                    ])
            
            # Next look for Breeze citations.
            if prow['reference'].strip():
                rvals = re.split(r"\]\s*,\s*\[", prow['reference'])
                refs = [
                    map(
                        lambda s:s.strip(), 
                        rv.strip(" []").split(',')
                        ) for rv in rvals ]
                prefs = dict([r[::-1] for r in refs])
            else:
                prefs = {}

            def getBreeze(refs):
                r = None
                for k, v in refs.items():
                    if k.startswith("Breeze"):
                        r = (k, v)
                        break
                return r

            breeze_ref = getBreeze(prefs)
            if breeze_ref:
                citations.append(
                    dict(
                        identifier=breeze_ref[1], 
                        range=breeze_ref[0], 
                        type="seeFurther") )

            field = place.getField('referenceCitations')
            field.resize(len(citations), place)
            place.update(referenceCitations=citations)

            LOG.info(
                "Citations: %s", 
                [c['range'] for c in place.getReferenceCitations()] )

            # Now we take care of the locations
            for lrow in locations:
                
                lid = place.invokeFactory(
                        'Location',
                        utils.normalizeString(lrow['title']),
                        title=lrow['title'],
                        description=lrow['description'],
                        geometry=lrow['geometry'],
                        associationCertainty=lrow['certainty_Loc'].replace(
                            " ", "-"),
                        creators=['sarcanon'],
                        contributors=['sgillies', 'thomase'],
                        originalProvenance='Pleiades' )

                location = place[lid]

                LOG.info(
                    "Created location: %s",
                    location.getId() )
                
                metadataDoc = context['features']['metadata'][
                    "hadrians-wall-milecastles-and-turrets-2012"]
                location.addReference(metadataDoc, 'location_accuracy')

                # Look for EH citations.
                citations = []
                if lrow['reference'].strip():
                    rvals = re.split(r"\]\s*,\s*\[", lrow['reference'])
                    refs = [map(
                            lambda s:s.strip(), 
                            rv.strip(" []").split(',')) for rv in rvals ]
                    lrefs = dict([r[::-1] for r in refs])
                else:
                    lrefs = {}

                def getEH(refs):
                    r = None
                    for k, v in refs.items():
                        if k.startswith("English"):
                            r = (k, v)
                            break
                    return r

                # Look for one locally...
                eh_ref = getEH(lrefs)
                if eh_ref is None:
                    # Check in the parent place.
                    eh_ref = getEH(prefs)

                if eh_ref:
                    citations.append(
                        dict(
                            identifier=eh_ref[1],
                            range=eh_ref[0], 
                            type="cites") )

                field = place.getField('referenceCitations')
                field.resize(len(citations), location)
                location.update(referenceCitations=citations)

                LOG.info(
                    "Citations: %s", 
                    [c['range'] for c in location.getReferenceCitations()] )

                attestations = location.getAttestations()
                attestations = [
                    {'timePeriod': 'roman', 'confidence': 'confident'},
                    {'timePeriod': 'late-antique', 'confidence': 'confident'} ]
                field = location.getField('attestations')
                field.resize(len(attestations), location)
                location.update(attestations=attestations)

                now = DateTime(datetime.datetime.now().isoformat())
                
                location.setModificationDate(now)
                repo.save(location, MESSAGE)
                wftool.doActionFor(location, action='submit')
                wftool.doActionFor(location, action='publish')
                location.reindexObject()

            place.setModificationDate(now)
            repo.save(place, MESSAGE)
            wftool.doActionFor(place, action='submit')
            wftool.doActionFor(place, action='publish')
            place.reindexObject()

            LOG.info("Finished place: %s", pid)

        # Bail
        # assert True == False

    except Exception, e:
        savepoint.rollback()
        LOG.exception("Rolled back after catching exception: %s" % e)
 
    transaction.commit()

if __name__ == '__main__':
    # Zopectl doesn't handle command line arguments well, necessitating quoting
    # like this:
    #
    # $ instance run 'names.py -f names.csv -m "Set all descriptions from names-up2.csv again" -j description'
    
    parser = OptionParser()
    parser.add_option(
        "-f", "--file", dest="filename",
        help="Input filename", metavar="FILE")
    parser.add_option(
        "-u", "--user", dest="user",
        help="Run script as user")

    opts, args = parser.parse_args(sys.argv[1:])
    filename = opts.filename
    f = open(filename, 'rb')
    reader = DictReader(f)
    site = app['plone']
    setup_cmfuid(site)
    secure(site, opts.user or 'admin')
    main(site, reader)
    app._p_jar.sync()