Merge remote-tracking branch 'refs/remotes/project-open-data/master'

Author: fb98

.gitignore

@@ -1,3 +1,4 @@
 .c9revisions
 *~
 node_modules
+*.pyc

Install/esri2open/__init__.py

@@ -1,2 +1,3 @@
-from esri2open import toOpen, writeFile, closeUp, closeJSON
-from prepare import prepareFile, prepareGeoJSON
+from esri2open import toOpen, writeFile, closeUp, closeJSON,closeTOPOJSON
+from prepare import prepareFile, prepareGeoJSON,prepareTOPO
+from utilities import getExt,getName

Install/esri2open/esri2open.py

@@ -33,19 +33,27 @@ def closeCSV(out):
     out[1].close()
     return True
 
+def closeTOPOJSON(out):
+    file = open(out['out'],'w')
+    out['topo'].dump(file)
+    file.close()
+    return True
+
 def closeUp(out,fileType):
     if fileType == "geojson":    
         return closeJSON(out)
     elif fileType == "csv":    
         return closeCSV(out)
-    if fileType == "json":    
+    elif fileType == "json":    
         return closeJSON(out)
+    elif fileType == "topojson":    
+        return closeTOPOJSON(out)
     else:
         return False
 
 #this is the meat of the function, we could put it into a seperate file if we wanted
-def writeFile(outArray,featureClass,fileType,includeGeometry, first=True):
-    parser = parse(outArray,featureClass,fileType,includeGeometry,first)
+def writeFile(outArray,featureClass,fileType,includeGeometry, first=True, outName = False):
+    parser = parse(outArray,featureClass,fileType,includeGeometry,first,outName)
     #wrap it in a try so we don't lock the database
     try:
         for row in parser.rows:
@@ -71,7 +79,7 @@ def toOpen(featureClass, outJSON, includeGeometry="geojson"):
         AddMessage("this filetype doesn't make sense")
         return
     #geojson needs geometry
-    if fileType=="geojson":
+    if fileType in ("geojson", "topojson"):
         includeGeometry="geojson"
     elif fileType=="sqlite":
         includeGeometry="well known binary"

Install/esri2open/parseGeometry.py

@@ -206,7 +206,8 @@ def getParseFunc(shpType, geo):
             fun = parseMultiPolygon
         else:
             fun = parseMultiPatch
-    if geo=="geojson":
-        return fun
-    elif geo=="well known text":
-        return getWKTFunc(fun)
+
+    if geo=="well known text":
+        return getWKTFunc(fun)
+    else:
+        return fun

Install/esri2open/parseRow.py

@@ -1,4 +1,4 @@
-from utilities import listFields, getShp, getOID, statusMessage, parseProp, makeInter
+from utilities import listFields, getShp, getOID, statusMessage, parseProp, makeInter,getName
 from arcpy import SpatialReference, SearchCursor  
 from parseGeometry import getParseFunc
 from json import dump
@@ -7,7 +7,7 @@
 wgs84="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119522E-09;0.001;0.001;IsHighPrecision"
 
 class parse:
-    def __init__(self,outFile,featureClass,fileType,includeGeometry, first=True):
+    def __init__(self,outFile,featureClass,fileType,includeGeometry, first=True, outName=False):
         self.outFile = outFile
         self.fileType = fileType
         #first we set put the local variables we'll need
@@ -33,6 +33,13 @@ def __init__(self,outFile,featureClass,fileType,includeGeometry, first=True):
             self.parse = self.parseJSON
         elif fileType=="sqlite":    
             self.parse = self.parseSqlite
+        elif fileType=="topojson":    
+            self.parse = self.parseTOPOJSON
+            if outName:
+                self.oName=outName
+            else:
+                self.oName = getName(featureClass)
+            self.topo = self.outFile['topo'].object_factory(self.oName)
 
     def cleanUp(self,row):
         del row
@@ -48,6 +55,9 @@ def parseCSV(self,row):
                 fc["geometry"]=self.parseGeo(row.getValue(self.shp))
             except:
                 return
+        for key in fc:
+            if isinstance(fc[key],unicode):
+                fc[key] = fc[key].encode('utf_8')
         self.outFile[0].writerow(fc)
 
     def parseGeoJSON(self,row):
@@ -73,6 +83,23 @@ def parseGeoJSON(self,row):
             self.outFile.write(",")
             dump(fc,self.outFile)
 
+    def parseTOPOJSON(self,row):
+        #more messages
+        self.status.update()
+        fc={"type": "Feature"}
+        if self.parseGeo:
+            try:
+                fc["geometry"]=self.parseGeo(row.getValue(self.shp))
+            except:
+                return
+        else:
+            raise NameError("we need geometry for topojson")
+        fc["id"]=row.getValue(self.oid)
+        fc["properties"]=parseProp(row,self.fields, self.shp)
+        if fc["geometry"]=={}:
+            return
+        self.topo(fc)
+
     def parseJSON(self,row):
         #more messages
         self.status.update()

Install/esri2open/prepare.py

@@ -2,6 +2,7 @@
 from utilities import listFields,getShp, parseFieldType
 from sqlite3 import Connection
 from os.path import splitext, split
+from topojson import Topology
 
 def prepareCSV(outJSON,featureClass,fileType,includeGeometry):
     shp=getShp(featureClass)[0]
@@ -58,6 +59,10 @@ def prepareJSON(outJSON,*args):
     out.write("""{"docs":[""")
     return out
 
+def prepareTOPO(outJSON,*args):
+    topo = Topology()
+    return {"topo":topo,'out':outJSON}
+
 def prepareFile(outJSON,featureClass,fileType,includeGeometry):
     if fileType == "geojson":    
         return prepareGeoJSON(outJSON,featureClass,fileType,includeGeometry)
@@ -67,5 +72,7 @@ def prepareFile(outJSON,featureClass,fileType,includeGeometry):
         return prepareJSON(outJSON,featureClass,fileType,includeGeometry)
     elif fileType == "sqlite": 
         return prepareSqlite(outJSON,featureClass,fileType,includeGeometry)
+    elif fileType == "topojson": 
+        return prepareTOPO(outJSON,featureClass,fileType,includeGeometry)
     else:
         return False

Install/esri2open/topojson/__init__.py

@@ -0,0 +1 @@
+from topology import Topology

Install/esri2open/topojson/arcs.py

@@ -0,0 +1,56 @@
+from hashtable import Hashtable
+from os import remove
+import shelve
+from tempfile import mkdtemp
+from hashlib import sha1
+from utils import point_compare
+class Arcs:
+    def __init__(self,Q):
+        self.coincidences = Hashtable(Q * 10)
+        self.arcsByPoint = Hashtable(Q * 10)
+        self.pointsByPoint = Hashtable(Q * 10)
+        self.arc_db_path=mkdtemp()+'/arc_db'
+        self.arcs= shelve.open(self.arc_db_path)
+        #self.arcs={}
+        self.length=0
+        self.storage_path = mkdtemp()+'/db'
+        self.db = shelve.open(self.storage_path)
+        #self.db={}
+    def get_index(self,point):
+        return self.pointsByPoint.get(point)
+    def get_point_arcs(self,point):
+        return self.arcsByPoint.get(point)
+    def coincidence_lines(self,point):
+        return self.coincidences.get(point)
+    def peak(self,point):
+        return self.coincidences.peak(point)
+    def push(self,arc):
+        self.arcs[str(self.length)]=arc
+        self.length+=1
+        return self.length
+    def close(self):
+        #pass
+        self.db.close()
+        remove(self.storage_path)
+        self.arcs.close()
+        remove(self.arc_db_path)
+    def get_hash(self,arc):
+        ourhash = sha1()
+        ourhash.update(str(arc))
+        return ourhash.hexdigest()
+    def check(self,arcs):
+        a0 = arcs[0]
+        a1 = arcs[-1]
+        point = a0 if point_compare(a0, a1) < 0 else a1
+        point_arcs = self.get_point_arcs(point)
+        h = self.get_hash(arcs)
+        if h in self.db:
+            return int(self.db[h])
+        else:
+            index = self.length
+            point_arcs.append(arcs)
+            self.db[h]=index
+            self.db[self.get_hash(list(reversed(arcs)))]=~index
+            self.push(arcs)
+            return index
+        

Install/esri2open/topojson/bounds.py

@@ -0,0 +1,20 @@
+from mytypes import Types
+class Bounds(Types):
+    def __init__(self):
+        self.x0=self.y0=float('inf')
+        self.x1=self.y1=-float('inf')
+    def point (self,point):
+        x = point[0]
+        y = point[1]
+        if x < self.x0:
+            self.x0 = x
+        if x > self.x1:
+            self.x1 = x
+        if y < self.y0:
+            self.y0 = y
+        if y > self.y1:
+            self.y1 = y
+def bound(objects):
+    b=Bounds()
+    b.obj(objects)
+    return b

Install/esri2open/topojson/clockwise.py

@@ -0,0 +1,25 @@
+class Clock:
+    def __init__(self,area):
+        self.area=area
+    def clock(self,feature):
+        if 'geometries' in feature:
+            feature['geometries'] = map(self.clock_geometry,feature['geometries'])
+        elif 'geometry' in feature:
+            feature['geometry']=self.clock_geometry(feature['geometry'])
+        return feature
+    def clock_geometry(self,geo):
+        if 'type' in geo:
+            if geo['type']=='Polygon' or geo['type']=='MultiLineString':
+                geo['coordinates'] = self.clockwise_polygon(geo['coordinates'])
+            elif geo['type']=='MultiPolygon':
+                geo['coordinates'] = map(lambda x:self.clockwise_polygon(x),geo['coordinates'])
+            elif geo['type']=='LineString':
+                geo['coordinates'] = self.clockwise_ring(geo['coordinates'])
+        return geo
+    def clockwise_polygon(self,rings):
+        return map(lambda x:self.clockwise_ring(x),rings)
+    def clockwise_ring(self,ring):
+        if self.area(ring) > 0:
+            return list(reversed(ring))
+        else:
+            return ring