diff --git a/.gitattributes b/.gitattributes index 65dbf4920..cb4038184 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,5 @@ # don't mess with line endings for Gedcom files *.ged binary *.GED binary + +*.csv -text diff --git a/data/tests/bug 8322 test.ged b/data/tests/bug_8322_test.ged similarity index 100% rename from data/tests/bug 8322 test.ged rename to data/tests/bug_8322_test.ged diff --git a/data/tests/bug 8322 test.gramps b/data/tests/bug_8322_test.gramps similarity index 100% rename from data/tests/bug 8322 test.gramps rename to data/tests/bug_8322_test.gramps diff --git a/data/tests/exp_sample_csv.csv b/data/tests/exp_sample_csv.csv new file mode 100644 index 000000000..4cc5b7abc --- /dev/null +++ b/data/tests/exp_sample_csv.csv @@ -0,0 +1,122 @@ +Person,Surname,Given,Call,Suffix,Prefix,Title,Gender,Birth date,Birth place,Birth source,Baptism date,Baptism place,Baptism source,Death date,Death place,Death source,Burial date,Burial place,Burial source,Note +[I0030],Adams,Janice Ann,,,,,female,1965-08-26,"Fremont, Alameda Co., CA",,,,,,,,,,, +[I0016],Anderson,Jennifer,,,,,female,1907-11-05,"Rønne, Bornholm, Denmark",,,,,1985-05-29,"San Francisco, San Francisco Co., CA",,,,, +[I0025],Ericsdotter,Marta,,,,,female,about 1775,Sweden,,,,,,,,,,, +[I0041],Green,Janis Elaine,,,,,female,1935-12-02,,,,,,,,,,,, +[I0000],Hansdotter,Anna,,,,,female,1864-10-02,"Löderup, Malmöhus Län, Sweden",,,,,1945-09-29,"Sparks, Washoe Co., NV",,,,, +[I0038],Hansdotter,Kerstina,,,,,female,1832-11-29,"Smestorp, Kristianstad Län, Sweden",,,,,before 1908,Sweden,,,,, +[I0032],Horne,Darcy,,,,,female,1966-07-02,"Sacramento, Sacramento Co., CA",,,,,,,,,,, +[I0036],Jefferson,Elna,,,,,female,1800-09-14,"Gladsax, Kristianstad Län, Sweden",,,,,,Sweden,,,,, +[I0017],Jones,Lillie Harriet,,,,,female,1910-05-02,"Rønne, Bornholm, Denmark",,,,,1990-06-26,,,,,, +[I0042],Ke 柯,雪,,,,,male,,,,,,,,,,,,, +[I0013],Michaels,Evelyn,,,,,female,about 1897,,,,,,,,,,,, +[I0012],Nielsen,Herman Julius,,,,,male,1889-08-31,"Rønne, Bornholm, Denmark",,,,,1945,,,,,, +[I0031],Ohman,Marjorie,,,,,female,1903-06-03,"Denver, Denver Co., CO, Denver Co., Colorado, USA",,,,,1980-06-22,"Reno, Washoe Co., NV",,,,, +[I0034],Perkins,Alice Paula,,,,,female,1933-11-22,"Sparks, Washoe Co., NV",,,,,,,,,,, +[I0002],Smith,Amber Marie,,,,,female,1998-04-12,"Hayward, Alameda Co., CA",,,,,,,,,,, +[I0023],Smith,Astrid Shermanna Augusta,,,,,female,1889-01-31,"Rønne, Bornholm, Denmark",,,,,1963-12-21,"San Francisco, San Francisco Co., CA",,,,, +[I0020],Smith,Carl Emil,,,,,male,1899-12-20,"Rønne, Bornholm, Denmark",,,,,1959-01-28,"Reno, Washoe Co., NV",,,,, +[I0029],Smith,Craig Peter,,,,,male,after 1966,"San Francisco, San Francisco Co., CA",,,,,,,,,,, +[I0037],Smith,Edwin Michael,,,,,male,1961-05-24,"San Jose, Santa Clara Co., CA","Birth, Death and Marriage Records",,,,,,,,,, +[I0009],Smith,Emil,,,,,male,1860-09-27,"Simrishamn, Kristianstad Län, Sweden",,,,,,,,,,, +[I0019],Smith,Eric Lloyd,,,,Dr.,male,1963-08-28,"San Francisco, San Francisco Co., CA",,,,,,,,,,, +[I0015],Smith,Gus,,,,,male,1897-09-11,"Rønne, Bornholm, Denmark",,,,,1963-10-21,"San Francisco, San Francisco Co., CA",,,,, +[I0024],Smith,Gustaf,,Sr.,,,male,1862-11-28,"Grostorp, Kristianstad Län, Sweden",,,,,before 1930-07-23,"Sparks, Washoe Co., NV",,,,, +[I0011],Smith,Hanna,,,,,female,1821-01-29,"Gladsax, Kristianstad Län, Sweden",,,,,,,,,,, +[I0010],Smith,Hans Peter,,,,,male,1904-04-17,"Rønne, Bornholm, Denmark",Birth Records,,,,1977-01-29,"San Francisco, San Francisco Co., CA",,1977-02-05,"San Francisco, San Francisco Co., CA",findagrave.com, +[I0021],Smith,Hjalmar,,,,,male,1893-01-31,"Rønne, Bornholm, Denmark",,,,,1894-09-25,"Rønne, Bornholm, Denmark",,,,, +[I0008],Smith,Hjalmar,,,,,male,1895-04-07,"Rønne, Bornholm, Denmark",,1895-06-03,"Rønne Bornholm, Denmark",,1975-06-26,"Reno, Washoe Co., NV",,,,, +[I0007],Smith,Ingar,,,,,female,after 1823,"Gladsax, Kristianstad Län, Sweden",,,,,,,,,,, +[I0027],Smith,Ingeman,,,,,male,about 1770,Sweden,,,,,,,,,,, +[I0004],Smith,Ingeman,,,,,male,1826-01-29,"Gladsax, Kristianstad Län, Sweden",,,,,,,,,,, +[I0018],Smith,John Hjalmar,,,,,male,1932-01-30,"San Francisco, San Francisco Co., CA",,,,,,,,,,, +[I0001],Smith,Keith Lloyd,,,,,male,1966-08-11,"San Francisco, San Francisco Co., CA",,,,,,,,,,, +[I0026],Smith,Kirsti Marie,,,,,female,1886-12-15,"Rønne, Bornholm, Denmark",,,,,1966-07-18,"San Francisco, San Francisco Co., CA",,,,, +[I0035],Smith,Lars Peter,,,,,male,1991-09-16,"Santa Rosa, Sonoma Co., CA",,,,,,,,,,, +[I0033],Smith,Lloyd,,,,,male,1935-03-13,"San Francisco, San Francisco Co., CA",,,,,,,,,,, +[I0003],Smith,Magnes,,,,,male,1858-10-06,"Simrishamn, Kristianstad Län, Sweden",,,,,1910-02-20,"Rønne, Bornholm, Denmark",,,,, +[I0040],Smith,Marjorie Alice,,,,,female,1960-02-05,"San Jose, Santa Clara Co., CA",,,,,,,,,,, +[I0014],Smith,Marjorie Lee,,,,,female,1934-11-04,"Reno, Washoe Co., NV",,,,,,,,,,, +[I0022],Smith,Martin,,,,,male,1830-11-19,"Gladsax, Kristianstad Län, Sweden",,1830-11-23,"Gladsax, Kristianstad Län, Sweden",,between 1899 and 1905,Sweden,,,,, +[I0039],Smith,Martin,,,,,male,between 1794 and 1796,"Tommarp, Kristianstad Län, Sweden",,,,,,Sweden,,,,, +[I0005],Smith,Mason Michael,,,,,male,1996-06-26,"Hayward, Alameda Co., CA",,,,,,,,,,, +[I0028],Streiffert,Anna,,,,,female,1860-09-23,"Hoya/Jona/Hoia, Sweden",,,,,1927-02-02,"Rønne, Bornholm, Denmark",,,,, +[I0006],Willard,Edwin,,,,,male,about 1886,,,,,,,,,,,, + +Marriage,Husband,Wife,Date,Place,Source,Note +[F0000],[I0039],[I0036],about 1816,"Gladsax, Kristianstad Län, Sweden",, +[F0001],[I0027],[I0025],about 1790,Sweden,, +[F0002],[I0022],[I0038],about 1856,,, +[F0003],[I0024],[I0000],1885-11-27,"Rønne, Bornholm, Denmark",, +[F0004],[I0006],[I0026],about 1910,,, +[F0005],[I0012],[I0023],1912-11-30,"Rønne, Bornholm, Denmark",, +[F0006],[I0008],[I0031],1927-10-31,"Reno, Washoe Co., NV",, +[F0007],[I0015],[I0013],about 1920,,, +[F0008],[I0033],[I0041],1958-08-10,"San Francisco, San Francisco Co., CA",, +[F0009],[I0010],[I0017],,,, +[F0010],[I0019],[I0032],1986-07-12,"Woodland, Yolo Co., CA",, +[F0011],[I0003],[I0028],1884-08-24,"Rønne, Bornholm, Denmark",, +[F0012],[I0018],[I0034],1954-06-04,"Sparks, Washoe Co., NV",Marriage Certificae, +[F0013],[I0037],[I0030],1995-05-27,"San Ramon, Conta Costa Co., CA",, +[F0014],[I0010],[I0016],,,, + +Family,Child +[F0000],[I0011] +[F0000],[I0007] +[F0000],[I0004] +[F0000],[I0022] +[F0001],[I0039] +[F0002],[I0003] +[F0002],[I0009] +[F0002],[I0024] +[F0003],[I0026] +[F0003],[I0023] +[F0003],[I0021] +[F0003],[I0008] +[F0003],[I0015] +[F0003],[I0020] +[F0003],[I0010] +[F0005],[I0042] +[F0006],[I0018] +[F0006],[I0014] +[F0008],[I0019] +[F0008],[I0001] +[F0008],[I0029] +[F0009],[I0033] +[F0010],[I0035] +[F0012],[I0040] +[F0012],[I0037] +[F0013],[I0005] +[F0013],[I0002] + +Place,Title,Name,Type,Latitude,Longitude,Code,Enclosed_by,Date +[P0000],"Löderup, Malmöhus Län, Sweden","Löderup, Malmöhus Län, Sweden",Unknown,,,,, +[P0001],"Sparks, Washoe Co., NV","Sparks, Washoe Co., NV",Unknown,,,,, +[P0002],"San Francisco, San Francisco Co., CA","San Francisco, San Francisco Co., CA",Unknown,,,,, +[P0003],"Rønne, Bornholm, Denmark","Rønne, Bornholm, Denmark",Unknown,,,,, +[P0004],"Gladsax, Kristianstad Län, Sweden","Gladsax, Kristianstad Län, Sweden",Unknown,,,,, +[P0005],"Reno, Washoe Co., NV","Reno, Washoe Co., NV",Unknown,,,,, +[P0006],"Hayward, Alameda Co., CA","Hayward, Alameda Co., CA",Unknown,,,,, +[P0007],"Community Presbyterian Church, Danville, CA","Community Presbyterian Church, Danville, CA",Unknown,,,,, +[P0008],Sweden,Sweden,Unknown,,,,, +[P0009],"Grostorp, Kristianstad Län, Sweden","Grostorp, Kristianstad Län, Sweden",Unknown,,,,, +[P0010],"Copenhagen, Denmark","Copenhagen, Denmark",Unknown,,,,, +[P0011],"Hoya/Jona/Hoia, Sweden","Hoya/Jona/Hoia, Sweden",Unknown,,,,, +[P0012],"Simrishamn, Kristianstad Län, Sweden","Simrishamn, Kristianstad Län, Sweden",Unknown,,,,, +[P0013],"Fremont, Alameda Co., CA","Fremont, Alameda Co., CA",Unknown,,,,, +[P0016],"Santa Rosa, Sonoma Co., CA","Santa Rosa, Sonoma Co., CA",Unknown,,,,, +[P0017],"San Jose, Santa Clara Co., CA","San Jose, Santa Clara Co., CA",Unknown,,,,, +[P0018],UC Berkeley,UC Berkeley,Unknown,,,,, +[P0019],"Smestorp, Kristianstad Län, Sweden","Smestorp, Kristianstad Län, Sweden",Unknown,,,,, +[P0020],"Tommarp, Kristianstad Län, Sweden","Tommarp, Kristianstad Län, Sweden",Unknown,,,,, +[P0021],"Rønne Bornholm, Denmark","Rønne Bornholm, Denmark",Unknown,,,,, +[P0022],"Woodland, Yolo Co., CA","Woodland, Yolo Co., CA",Unknown,,,,, +[P0023],"San Ramon, Conta Costa Co., CA","San Ramon, Conta Costa Co., CA",Unknown,,,,, +[P0026],United States of America,USA,Country,,,,, +[P0028],"California, USA",California,State,,,,[P0026],2016-06-04 +[P0025],"Colorado, USA",Colorado,State,,,,[P0026], +[P0027],"Sacramento Co., California, USA",Sacramento Co.,County,,,,[P0028],2016-06-01 +[P0015],"Sacramento, Sacramento Co., CA","Sacramento, Sacramento Co., CA",City,,,,[P0027],2016-06-04 +[P0024],"Denver Co., Colorado, USA",Denver Co.,County,,,,[P0025], +[P0014],"Denver, Denver Co., CO","Denver, Denver Co., CO",City,39.7392,104.9903 W,,[P0024], + diff --git a/data/tests/exp_sample_csv.gramps b/data/tests/exp_sample_csv.gramps new file mode 100644 index 000000000..4f4a7e8ce --- /dev/null +++ b/data/tests/exp_sample_csv.gramps @@ -0,0 +1,1370 @@ + + + +
+ + + Alex Roitman,,, + Not Provided + +
+ + + + + + Birth + + + Birth of Anna Hansdotter + + + Death + + + Death of Anna Hansdotter + + + Birth + + + Birth of Keith Lloyd Smith + + + Birth + + + Birth of Hans Peter Smith + + + + Death + + + Death of Hans Peter Smith + + + Birth + + + Birth of Hanna Smith + + + Birth + + + Birth of Herman Julius Nielsen + + + Death + + Death of Herman Julius Nielsen + + + Birth + + Birth of Evelyn Michaels + + + Birth + + + Birth of Marjorie Lee Smith + + + Birth + + + Birth of Gus Smith + + + Death + + + Death of Gus Smith + + + Birth + + + Birth of Jennifer Anderson + + + Death + + + Death of Jennifer Anderson + + + Birth + + + Birth of Lillie Harriet Jones + + + Death + + Death of Lillie Harriet Jones + + + Birth + + + Birth of John Hjalmar Smith + + + Birth + + + Birth of Eric Lloyd Smith + + + Adopted + + + Birth + + + Birth of Amber Marie Smith + + + Christening + + + Christening of Amber Marie Smith + + + Birth + + + Birth of Carl Emil Smith + + + Death + + + Death of Carl Emil Smith + + + + Birth + + + Birth of Hjalmar Smith + + + Death + + + Death of Hjalmar Smith + + + Birth + + + Birth of Martin Smith + + + Death + + + Death of Martin Smith + + + Baptism + + + Baptism of Martin Smith + + + Birth + + + Birth of Astrid Shermanna Augusta Smith + + + Death + + + Death of Astrid Shermanna Augusta Smith + + + Birth + + + Birth of Gustaf Smith, Sr. + + + Death + + + Death of Gustaf Smith, Sr. + + + Immi + + + + + Christening + + + Christening of Gustaf Smith, Sr. + + + Birth + + + Birth of Marta Ericsdotter + + + Birth + + + Birth of Kirsti Marie Smith + + + Death + + + Death of Kirsti Marie Smith + + + Birth + + + Birth of Ingeman Smith + + + Birth + + + Birth of Anna Streiffert + + + Death + + + Death of Anna Streiffert + + + Birth + + + Birth of Craig Peter Smith + + + Census + Census of Craig Peter Smith + + + + Birth + + + Birth of Magnes Smith + + + Death + + + Death of Magnes Smith + + + Birth + + + Birth of Janice Ann Adams + + + Occupation + Retail Manager + + + Degree + + Business Management + + + Birth + + + Birth of Marjorie Ohman + + + Death + + + Death of Marjorie Ohman + + + Birth + + + Birth of Darcy Horne + + + Birth + + + Birth of Lloyd Smith + + + Birth + + + Birth of Alice Paula Perkins + + + Birth + + + Birth of Lars Peter Smith + + + Adopted + + + Birth + + + Birth of Elna Jefferson + + + Death + + Death of Elna Jefferson + + + Christening + + + Christening of Elna Jefferson + + + Birth + + + Birth of Edwin Michael Smith + + + + Occupation + Software Engineer + + + + Education + + + Education of Edwin Michael Smith + + + Degree + + B.S.E.E. + + + Birth + + + Birth of Kerstina Hansdotter + + + Death + + + Death of Kerstina Hansdotter + + + Birth + + + Birth of Martin Smith + + + Death + + Death of Martin Smith + + + Birth + + + Birth of Ingeman Smith + + + Birth + + + Birth of Marjorie Alice Smith + + + Birth + + Birth of Janis Elaine Green + + + Birth + + + Birth of Mason Michael Smith + + + Christening + + + Christening of Mason Michael Smith + + + Birth + + Birth of Edwin Willard + + + Birth + + + Birth of Ingar Smith + + + Birth + + + Birth of Hjalmar Smith + + + Death + + + Death of Hjalmar Smith + + + Baptism + + + Baptism of Hjalmar Smith + + + Immi + + + + + Birth + + + Birth of Emil Smith + + + Marriage + + + Marriage of Martin Smith and Elna Jefferson + + + Marriage + + + Marriage of Ingeman Smith and Marta Ericsdotter + + + Marriage + + + Marriage of Eric Lloyd Smith and Darcy Horne + + + Marriage + + + Marriage of Magnes Smith and Anna Streiffert + + + Marriage + + + Marriage of John Hjalmar Smith and Alice Paula Perkins + + + + Marriage + + + Marriage of Edwin Michael Smith and Janice Ann Adams + + + Engagement + + + Engagement of Edwin Michael Smith and Janice Ann Adams + + + Marriage + + Marriage of Martin Smith and Kerstina Hansdotter + + + Marriage + + + Marriage of Gustaf Smith, Sr. and Anna Hansdotter + + + Marriage + + Marriage of Edwin Willard and Kirsti Marie Smith + + + Marriage + + + Marriage of Herman Julius Nielsen and Astrid Shermanna Augusta Smith + + + Marriage + + + Marriage of Hjalmar Smith and Marjorie Ohman + + + Marriage + + Marriage of Gus Smith and Evelyn Michaels + + + Marriage + + + Marriage of Lloyd Smith and Janis Elaine Green + + + Burial + + + In cemetary + + + + + + F + + Anna + Hansdotter + + + + + + + M + + Keith Lloyd + Smith + + + + + + M + + Hans Peter + Smith + + + + + + + + + + F + + Hanna + Smith + + + + + + M + + Herman Julius + Nielsen + + + + + + + F + + Evelyn + Michaels + + + + + + F + + Marjorie Lee + Smith + + + + + + M + + Gus + Smith + + + + + + + + F + + Jennifer + Anderson + + + + + + + F + + Lillie Harriet + Jones + + + + + + + M + + John Hjalmar + Smith + + + + + + + M + + Eric Lloyd + Smith + Dr. + + + + + + + + F + + Amber Marie + Smith + + + + + + + M + + Carl Emil + Smith + + + + + + + M + + Hjalmar + Smith + + + + + + + M + + Martin + Smith + + + + + + + + + + F + + Astrid Shermanna Augusta + Smith + + + + + + + + M + + Gustaf + Smith + Sr. + + + + + + + + + + F + + Marta + Ericsdotter + + + + + + F + + Kirsti Marie + Smith + + + + + + + + M + + Ingeman + Smith + + + + + + F + + Anna + Streiffert + + + + + + + M + + Craig Peter + Smith + + + + + + + M + + Magnes + Smith + + + + + + + + F + + Janice Ann + Adams + + + + + + + + F + + Marjorie + Ohman + + + + + + + F + + Darcy + Horne + + + + + + M + + Lloyd + Smith + + + + + + + F + + Alice Paula + Perkins + + + + + + M + + Lars Peter + Smith + + + + + + + F + + Elna + Jefferson + + + + + + + + M + + Edwin Michael + Smith + + + + + + + + + + + + + F + + Kerstina + Hansdotter + + + + + + + M + + Martin + Smith + + + + + + + + M + + Ingeman + Smith + + + + + + F + + Marjorie Alice + Smith + + + + + + F + + Janis Elaine + Green + + + + + + M + + Mason Michael + Smith + + + + + + + M + + Edwin + Willard + + + + + + F + + Ingar + Smith + + + + + + M + + Hjalmar + Smith + + + + + + + + + + + M + + Emil + Smith + + + + + + M + + 雪 + Ke 柯 + + + Frank + Neilsen + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2 + + + + 2 + + + + 2 + + + + + 3 + + + + + 22 6 + 2 + + + + + + Birth Certificate + + + Birth, Death and Marriage Records + + + + + Marriage Certificae + + + + + Birth Records + + + findagrave.com + + + + + Löderup, Malmöhus Län, Sweden + + + + + Sparks, Washoe Co., NV + + + + San Francisco, San Francisco Co., CA + + + + Rønne, Bornholm, Denmark + + + + Gladsax, Kristianstad Län, Sweden + + + + + Reno, Washoe Co., NV + + + + Hayward, Alameda Co., CA + + + + Community Presbyterian Church, Danville, CA + + + + Sweden + + + + Grostorp, Kristianstad Län, Sweden + + + + + Copenhagen, Denmark + + + + Hoya/Jona/Hoia, Sweden + + + + Simrishamn, Kristianstad Län, Sweden + + + + + Fremont, Alameda Co., CA + + + + Denver, Denver Co., CO + + + + + + Sacramento, Sacramento Co., CA + + + + + + + Santa Rosa, Sonoma Co., CA + + + + San Jose, Santa Clara Co., CA + + + + UC Berkeley + + + + Smestorp, Kristianstad Län, Sweden + + + + Tommarp, Kristianstad Län, Sweden + + + + + Rønne Bornholm, Denmark + + + + Woodland, Yolo Co., CA + + + + San Ramon, Conta Costa Co., CA + + + + Denver Co., Colorado, USA + + + + + Colorado, USA + + + + + United States of America + + + + Sacramento Co., California, USA + + + + + + + California, USA + + + + + + + + + + + + + + New York Public Library + Library +
+ 5th Ave at 42 street + New York + New York + USA + 11111 +
+ + + Aunt Martha's Attic + Library +
+ 123 Main St + Someville + ST + USA +
+ + + + + + + BIOGRAPHY +Martin was listed as being a Husman, (owning a house as opposed to a farm) in the house records of Gladsax. + + + Witness name: John Doe +Witness comment: This is a simple test. + + + Witness name: No Name + + + BIOGRAPHY + +Hjalmar sailed from Copenhagen, Denmark on the OSCAR II, 14 November 1912 arriving in New York 27 November 1912. He was seventeen years old. On the ship passenger list his trade was listed as a Blacksmith. He came to Reno, Nevada and lived with his sister Marie for a time before settling in Sparks. He worked for Southern Pacific Railroad as a car inspector for a time, then went to work for Standard Oil +Company. He enlisted in the army at Sparks 7 December 1917 and served as a Corporal in the Medical Corp until his discharge 12 August 1919 at the Presidio in San Francisco, California. Both he and Marjorie are buried in the Masonic Memorial Gardens Mausoleum in Reno, he the 30th June 1975, and she the 25th of June 1980. + + + But Aunt Martha still keeps the original! + + + The repository reference from the source is important + + + Some note on the repo + + + Some Unicode Characters: ࣶǼЀج⿄㑝㵋圛墉幵聟聦𐅉🚶🛈 +Some Bold Unicode Characters: ࣶǼЀج⿄㑝㵋圛墉幵聟聦𐅉🚶🛈 +Some Italic Unicode Characters: ࣶǼЀج⿄㑝㵋圛墉幵聟聦𐅉🚶🛈 + + + + + diff --git a/gramps/plugins/export/exportcsv.py b/gramps/plugins/export/exportcsv.py index 6329c87ff..06f75584a 100644 --- a/gramps/plugins/export/exportcsv.py +++ b/gramps/plugins/export/exportcsv.py @@ -58,6 +58,7 @@ from gramps.gen.utils.string import gender as gender_map from gramps.gen.datehandler import get_date from gramps.gen.display.place import displayer as _pd from gramps.gui.glade import Glade +from gramps.gen.constfunc import win #------------------------------------------------------------------------- # @@ -102,72 +103,6 @@ def get_primary_source_title(db, obj): return source.get_title() return "" -#------------------------------------------------------------------------- -# -# Encoding support for CSV, from http://docs.python.org/lib/csv-examples.html -# -#------------------------------------------------------------------------- -class UTF8Recoder: - """Iterator that reads an encoded stream and reencodes the input to UTF-8.""" - def __init__(self, f, encoding): - self.reader = codecs.getreader(encoding)(f) - - def __iter__(self): - return self - - def __next__(self): - return self.reader.next().encode("utf-8") - -class UnicodeReader: - """ - A CSV reader which will iterate over lines in the CSV file "f", which is - encoded in the given encoding. - - """ - - def __init__(self, f, encoding="utf-8", **kwds): - f = UTF8Recoder(f, encoding) - self.reader = csv.reader(f, **kwds) - - def __next__(self): - row = next(self.reader) - return [str(s, "utf-8") for s in row] - - def __iter__(self): - return self - -class UnicodeWriter: - """ - A CSV writer which will write rows to CSV file "f", which is encoded in - the given encoding. - - """ - - def __init__(self, f, encoding="utf-8", **kwds): - # Redirect output to a queue - self.queue = StringIO() - self.writer = csv.writer(self.queue, **kwds) - self.stream = f - self.encoder = codecs.getencoder(encoding) - - def writerow(self, row): - self.writer.writerow(row) - data = self.queue.getvalue() - #data now contains the csv data in unicode - # ... and reencode it into the target encoding - data, length = self.encoder(data) - # write to the target stream - self.stream.write(data) - # empty queue, go to start position, then truncate - self.queue.seek(0) - self.queue.truncate(0) - - def writerows(self, rows): - list(map(self.writerow, rows)) - - def close(self): - self.stream.close() - #------------------------------------------------------------------------- # # CSVWriter Options @@ -270,7 +205,7 @@ class CSVWriter: # make place list so that dependencies are first: self.place_list = [] - place_list = [x for x in self.db.iter_place_handles()] + place_list = sorted([x for x in self.db.iter_place_handles()]) while place_list: handle = place_list[0] place = self.db.get_place_from_handle(handle) @@ -317,9 +252,10 @@ class CSVWriter: def export_data(self): self.dirname = os.path.dirname (self.filename) try: - self.g = open(self.filename,"w") - self.fp = open(self.filename, "wb") - self.g = UnicodeWriter(self.fp) + self.fp = open(self.filename, "w", + encoding='utf_8_sig' if win() else 'utf_8', + newline='') + self.g = csv.writer(self.fp) except IOError as msg: msg2 = _("Could not create %s") % self.filename self.user.notify_error(msg2,str(msg)) @@ -594,7 +530,7 @@ class CSVWriter: place_latitude, place_longitude, place_code, "", "") self.writeln() - self.g.close() + self.fp.close() return True def format_date(self, date): diff --git a/gramps/plugins/importer/importcsv.py b/gramps/plugins/importer/importcsv.py index f9c677543..b418ad595 100644 --- a/gramps/plugins/importer/importcsv.py +++ b/gramps/plugins/importer/importcsv.py @@ -32,6 +32,7 @@ import time import csv import codecs +from io import TextIOWrapper #------------------------------------------------------------------------ # @@ -106,7 +107,16 @@ def importData(dbase, filename, user): parser = CSVParser(dbase, user, (config.get('preferences.tag-on-import-format') if config.get('preferences.tag-on-import') else None)) try: - with open(filename, 'r') as filehandle: + with open(filename, 'rb') as filehandle: + line = filehandle.read(3) + if line == codecs.BOM_UTF8: + filehandle.seek(0) + filehandle = TextIOWrapper(filehandle, encoding='utf_8_sig', + errors='replace', newline='') + else: # just open with OS encoding + filehandle.seek(0) + filehandle = TextIOWrapper(filehandle, + errors='replace', newline='') parser.parse(filehandle) except EnvironmentError as err: user.notify_error(_("%s could not be opened\n") % filename, str(err)) diff --git a/gramps/plugins/test/test_exports.py b/gramps/plugins/test/test_exports.py new file mode 100644 index 000000000..f31eabc14 --- /dev/null +++ b/gramps/plugins/test/test_exports.py @@ -0,0 +1,111 @@ +#! /usr/bin/env python3 +""" +Gramps - a GTK+/GNOME based genealogy program + +Copyright (c) 2016 Gramps Development Team + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +""" + +import unittest +import os +import difflib + +from gramps.test.test_util import Gramps +from gramps.gen.const import TEMP_DIR, DATA_DIR +from gramps.gen.datehandler import set_format +from gramps.cli.user import User + +TREE_NAME = "Test_exporttest" +TEST_DIR = os.path.abspath(os.path.join(DATA_DIR, "tests")) + + +def call(*args): + """ Call Gramps to perform the action with out and err captured """ + print("call:", args) + set_format(0) # Use ISO date for test + gramps = Gramps(user=User(auto_accept=True, quiet=True)) + out, err = gramps.run(*args) + print("out:", out, "err:", err) + return out, err + + +def do_it(tstfile): + """ based on tstfile, prepare an result export and compare with + expected. + """ + fname = os.path.splitext(os.path.basename(tstfile))[0] + + tst_file = os.path.join(TEST_DIR, fname + ".gramps") + expect_file = os.path.join(TEST_DIR, tstfile) + result_file = os.path.join(TEMP_DIR, tstfile) + err = call("-C", TREE_NAME, "-q", + "--import", tst_file, + "--export", result_file)[1] + if "Cleaning up." not in err: + return "Export failed, no 'Cleaning up.'" + msg = compare(expect_file, result_file) + if not msg: + # we will leave the result_file in place if there was an error. + try: + os.remove(result_file) + except OSError: + pass + return + else: + return msg + + +def compare(expect_file, result_file): + """ This uses the diff library to compare two files + """ + with open(expect_file, encoding='utf-8_sig') as exp_f, \ + open(result_file, encoding='utf-8_sig') as res_f: + diff = difflib.unified_diff(exp_f.readlines(), + res_f.readlines(), + n=2, lineterm='\n') + msg = "" + for line in diff: + if line == "--- \n" or line == "+++ \n": + continue + msg += line + return msg + + +class ExportControl(unittest.TestCase): + """ These tests compare various exported files with expected files, + based on the matching '.gramps' test file as a source. + As more types of exports are tested, we will need to provide some + filters for the differences; some types of exports have Gramps versions, + export dates, file names etc. that don't count as differences. + """ + def setUp(self): + self.tearDown() # removes it if it existed + +# out, err = self.call("-C", TREE_NAME, +# "--import", example) + + def tearDown(self): + call("-y -q", "--remove", TREE_NAME) + + def test_csv(self): + """ Run a csv export test """ + tst_file = 'exp_sample_csv.csv' + msg = do_it(tst_file) + if msg: + self.fail(tst_file + ': ' + msg) + +if __name__ == "__main__": + unittest.main() diff --git a/gramps/plugins/test/test_imports.py b/gramps/plugins/test/test_imports.py index 02f81b44b..0bcd44a8c 100644 --- a/gramps/plugins/test/test_imports.py +++ b/gramps/plugins/test/test_imports.py @@ -243,7 +243,8 @@ else: _tstfiles = [] for _tstfile in os.listdir(TEST_DIR): (fname, ext) = os.path.splitext(os.path.basename(_tstfile)) - if ext == ".gramps" or ext == ".difs" or ext == ".bak": + if ext == ".gramps" or ext == ".difs" or ext == ".bak" \ + or fname.startswith("exp_"): continue test_func = make_tst_function(_tstfile, fname) clname = 'Import_{0}'.format(_tstfile)