From 1572df187cda789f733532d3720fa3920f6bd7c7 Mon Sep 17 00:00:00 2001
From: Fernando Serboncini <fserb@fserb.com.br>
Date: Tue, 7 Dec 2010 02:10:21 +0100
Subject: [PATCH 1/2] New test data with unicode strings

---
 test/data/sample-feed.zip | Bin 3249 -> 3261 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/test/data/sample-feed.zip b/test/data/sample-feed.zip
index 385ebc9a83815ea5b8eb03392f7bc519b92866a2..2ab7db188c600b89c05a53053b4d419c23e1b967 100644
GIT binary patch
delta 705
zcmdlexmS`mz?+#xgn@y9gP~VVWFqfQ=Cf)dlM5JIn2!LtCzmi^1#-VLu4MvJ%b1pe
zC|~9ojJqd4V_pQ}bg`_j7Zhx_o$_!$M+_qaLk}|p11EzFLvcxdL9t#*MM-D~Cj+x!
z=U*QXF0J5ZU}Sj#R0Af~PCo0`Vj$A`-NrS1T~_-vCfV|1Tea^@=8|2`xvk}1;}5&M
z+OA4N&DMzmvavfJ6<jXfH|L+lhTN6eZ~k0A5iT0CHbyi=xjs?s8cX3TsXI5knzk)#
z*&g;Vtn8G-%Y<1AS1kAuX#KpDk>&0JC-c3pF4V5)sI$^~Qaqt7qbtyKQp17%x14>=
zMGacFXI@O;zZ@eSFgJLrzq-Xq&vP!*0&2V^m2DRK81yV|%3Z;~t=VnXuXMfci;kKc
z3I5TnJ~DofQ;r!NV^*ubnIIv*_udEnvq8tiltfo9n0I2kixt~nCzIr#_Cm$KT<^5^
zy=?P1I<-%Scelkww==05{m-vR_^nzuzvV}c^!&Ka<)0OOd-qu{(K#K~VWUvAXU^%Y
zza{n`H=VcHcjZ}TkD;DRNXOLoe>O^-e}8?Bd&9T9?b2<HHmCVcbiXbCoUB^VU19k$
zO4q)uhIen6^T{2|O0nIVZWq-NQM(NoQ|H;9fFf)wdk2$-$Ygtt4iN7ohc7t7vpJ=i
z_b@O_ZeSFhJdaZe6#0`SIi;DL87G@@8i5nRVa{-n4jHZ>5LLhx3`_(^xcosJQEp#e
zlteH&pIaFe>5~JwWaNRC+s@c@YtB4iD$4;T5k3Y9Ad7(k$Yfa3cy96qZZkQc66S*z
l70y7RFd*hdQL<&S3Xdts(a}88jCPaDc}&=jaR5~@002)k1+xGE

delta 709
zcmdlhxlxigz?+#xgn@y9gQ0hx@I>C7%tz)4PYz&Q!h9CUJ=wy170CV0xRwb>En`{=
zqI{WWFcwdK#=Ho`>0()5za&(|w)1=5+5kodhAw6X22KVUhT@X^f?~arijvR}P6lS>
z{dqnhTw1}+z{v6fs0K`|o$Q$3q9D-v{vVg0t#7Pj$hPgvp3i%8Hz2j#Q%7Rv)i3wM
zUA9YK&bz%tT~7E}Z6%Mzme{4z1$CyMd|9t*gtK4uifJ)*te^8+EGpXj%C(Rm_cULm
ztmg^idoI=K#dZFc<$CX)ld_tn)+^@O@M*P8pR9DRJ;Oas`l>)DOYO~hA5FB{mu%ZR
zQ|e9Py6omv<*UBEGhE^tV=!^K$UDtsLw`+S9tEwNZNE~gbCzcAd%mefX38OkZ(FpR
zXCBU+q$hE#;=mGVp2xxU>+^q1PmxU4p4jCR;Qu6AX>L!Q!Ys`{`kkzQRo*!t%W+=v
z$jxn9YxJB<;b%oRc0Ui<@HcC}e{=1=t>^P@Tu${UUbMb#%JZM1i#R%%uI-GOJO94=
zn(t>Od*{qkWeDh6AyWU&TCt3O_g#y=kE`Dp+TBXrUc<V*>v5!c*4-N~^?4+tdPN`e
zCBL*<;IN1v5uMwBv2&a42`G{dvUe~o5T5J}G#|*j%Ha!+<XTQ?racUkXK@-S0OQ)0
zEn1?$8ki8mfLMq@0!T7203(!PNn<kOWG7Awa7sAO84fZ+gDVI`)o=v^Q^Ex<e-KBJ
z+m{z5B}}g7R%TM(KbetRmKo%j1KiS#hbG_QHj@L|#(dDC!Wn2f$TnUS+g40A;4zh3
fAZ*)l#AK5zP&yWfIZ>1tO>X8fVcW$4lwbe=dV2`~


From e0390740ef951bebc2e1a5448abc68e51891833d Mon Sep 17 00:00:00 2001
From: Fernando Serboncini <fserb@fserb.com.br>
Date: Tue, 7 Dec 2010 02:15:31 +0100
Subject: [PATCH 2/2] fixed unicode support

---
 gtfs/entity/unmapped_entities.py |  4 +++-
 gtfs/feed.py                     | 19 ++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/gtfs/entity/unmapped_entities.py b/gtfs/entity/unmapped_entities.py
index 78b0e44..3dcabc4 100644
--- a/gtfs/entity/unmapped_entities.py
+++ b/gtfs/entity/unmapped_entities.py
@@ -7,7 +7,9 @@ def __init__(self, **kwargs):
         attrvaluestr = kwargs[attrname]
 	if attrvaluestr == '':
 	  attrvalue = None
-	else:
+	elif attrtype == str:
+          attrvalue = unicode(attrvaluestr)
+        else:
 	  attrvalue = attrtype( attrvaluestr )
 
         setattr( self, attrname, attrvalue )
diff --git a/gtfs/feed.py b/gtfs/feed.py
index 584f1c4..67fe74f 100644
--- a/gtfs/feed.py
+++ b/gtfs/feed.py
@@ -11,7 +11,8 @@ def __init__(self,header,row):
     self.row = row 
 
   def to_dict(self):
-    return dict([(fieldname,self.row[fieldindex] if fieldindex<len(self.row) else None) for fieldname,fieldindex in self.header.items()])
+    return dict([(fieldname,self.row[fieldindex] if fieldindex<len(self.row) 
+                  else None) for fieldname,fieldindex in self.header.items()])
 
   def __repr__(self):
     return repr(self.to_dict())
@@ -41,8 +42,8 @@ def next(self):
     return Record( self.header, self.rows.next() )
 
 class Feed(object):
-  """A Feed is a collection of CSV files with headers, either zipped into an archive
-     or loose in a folder"""
+  """A Feed is a collection of CSV files with headers, either zipped into an
+  archive or loose in a folder"""
 
   def __init__(self, filename):
     self.filename = filename 
@@ -51,15 +52,23 @@ def __init__(self, filename):
     if not os.path.isdir( filename ):
       self.zf = ZipFile( filename )
 
+  def unicode_csv_reader(self, content_iter):
+    reader = csv.reader( [ x.encode("utf-8") for x in iterdecode(content_iter,
+                                                                 "utf-8") ] )
+    for row in reader:
+      yield [ unicode(x, "utf-8") for x in row ]
+    return
+
   def get_rows(self, filename):
     if self.zf:
       try:
         contents = self.zf.read(filename)
       except KeyError:
         raise KeyError( "%s is not present feed"%filename )
-      return csv.reader( iterdecode( contents.split("\n"), "utf-8" ) )
+      return self.unicode_csv_reader( contents.split("\n") )
     else:
-      return csv.reader( iterdecode( open( os.path.join( self.filename, filename ) ), "utf-8" ) )
+      return self.unicode_csv_reader( open( os.path.join( self.filename,
+                                                          filename ) ) )
 
   def get_table(self, filename):
     rows = self.get_rows( filename )