re-fix relative path + tidy readme

.gitignore

@@ -49,3 +49,4 @@ redatamguiwindows/__/gui/moc_mainwindow.cpp_parameters
 redatamguiwindows/gui/moc_mainwindow.cpp_parameters
 dev/cpp11-0.5.0
 dev/pybind11-2.13.6
+dev/redatam-microdata

README.md

@@ -101,48 +101,6 @@ cmake --build . --config Release
 cd ..
 ```
 
-## Testing
-
-### Reading
-
-Ticked = Passed; Blank = Failed
-
-#### DIC format
-
-- [x] Argentina 1991 (`CP1991ARG/datos/Arg91CPV.dic`)
-- [x] Argentina 2001 (`CP2001ARG-DATA/CpvAr01_pub_v12.dic`)
-- [x] Argentina 2010 (`CP2010ARG/BASE_AMP_DPTO/CPV2010Ampliado.dic`)
-- [x] Bolivia 2001 (`CP2001BOL/Cp2001BOL/BaseOriginal/CPV2001.dic`)
-- [x] Bolivia 2012 (`CP2012BOL/BaseMunicipio_V3/CPV2012Municipio.dic`)
-- [x] Chile 2017 (`CP2017CHL/BaseOrg16/CPV2017-16.dic`)
-- [x] Dominican Republic 2002 (`CP2002DOM/Cp2002DOM/BaseOriginal/CPV2002DOM.dic`)
-- [x] Ecuador 2010 (`CP2010ECU/Base/CE11.dic`)
-- [x] Ecuador (Galapagos) 2015 (`test/galapagos/cg15.dic`)
-- [x] El Salvador 2007 (`CP2007SLV/CP2007SLV/BaseTotal/CPV2007ES.dic`)
-- [x] Guatemala 2018 (`CP2018GTM/BasePub/CPV2018GT_BasePublica.dic`)
-- [x] Mexico 2000 (`CP2000MEX/Cp2000MEX/BaseOriginal/cpmx2000.dic`)
-- [ ] Mexico 2010 (`CP2010MEX/BasePubM/MC10.dic`)
-- [x] Myanmar 2014 (`CP2014MMR/Union.dic`)
-- [x] Peru 2007 (`CP2007PER/CP2007PER/BasePub/CPV2007PER_PUB.dic`)
-- [x] Peru 2017 (`CP2017PER/BaseD/BaseR/CPVPER2017D.dic`)
-- [x] Uruguay 2011 (`CP2000MEX/Cp2000MEX/BaseOriginal/cpmx2000.dic`)
-
-#### DICX format
-
-- [x] Argentina 2010 (`CP2010ARG/BASE_AMP_DPTO/CPV2010Ampliado.dicx`)
-- [x] Bolivia 2001 (`CP2001BOL/Cp2001BOL/BaseOriginal/CPV2001.dicx`)
-- [x] Bolivia 2012 (`CP2012BOL/BaseMunicipio_V3/CPV2012Comunidad.dicx`)
-- [x] Chile 2017 (`CP2017CHL/BaseOrg16/CPV2017-16.dicx`)
-- [x] Dominican Republic 2002 (`CP2002DOM/Cp2002DOM/BaseOriginal/CPV2002DOM.dicx`)
-- [x] Ecuador 2010 (`CP2010ECU/Base/cpv2010ecu.dicx`)
-- [x] Ecuador (Galapagos) 2015 (`test/galapagos/cg15.dicX`)
-- [x] El Salvador 2007 (`CP2007SLV/CP2007SLV/BaseTotal/CPV2007ES.dicx`)
-- [x] Mexico 2000 (`CP2000MEX/Cp2000MEX/BaseOriginal/cpmx2000.dicx`)
-- [x] Mexico 2010 (`CP2010MEX/BasePubM/MC10.dicx`)
-- [x] Mexico 2015 (`ECI2015MEX/BaseR/ECI2015MX.dicX`)
-- [x] Peru 2007 (`CP2007PER/CP2007PER/BasePub/CPV2007PER_PUB.dicx`)
-- [x] Uruguay 2011 (`CP2011URY/CP2011URY/BaseRPub/CPV2011_uruguay_publica.dicX`)
-
 ### Validation against IPUMS data
 
 A simple validation exercise is to compare counts and percentages by sex and age groups against the [IPUMS](https://international.ipums.org/international/index.shtml) data to determine if the data was converted correctly.

dev/08-export-microdata.R

@@ -0,0 +1,66 @@
+devtools::load_all("./rpkg")
+
+dout <- "dev/redatam-microdata"
+
+if (!dir.exists(dout)) dir.create(dout)
+
+finp <- list.files("downloads/redatam", pattern = "\\.dic", full.names = TRUE, recursive = TRUE)
+
+finp <- finp[!finp %in%
+  c("downloads/redatam/ECI2015MEX/BaseR/ECI2015MX.dicX",
+  "downloads/redatam/CP2000BRA/BaseOriginal/CD102_PUBLICO.dic",
+  "downloads/redatam/CP2001BOL/Cp2001BOL/BaseOriginal/CPV2001.dic",
+  "downloads/redatam/CP2010MEX/BasePubM/MC10.dic")
+]
+
+finp <- finp[!grepl("BRA", finp)]
+finp <- finp[!grepl("Celade|celade", finp)]
+
+purrr::map(
+  finp,
+  function(f) {
+    print(f)
+
+    dout2 <- gsub(basename(f), "", f)
+    dout2 <- gsub("downloads/redatam", dout, dout2)
+
+    if (!dir.exists(dout2)) {
+      dir.create(dout2, recursive = TRUE)
+    } else {
+      return(TRUE)
+    }
+
+    d <- read_redatam(f)
+
+    fout <- paste0(dout2, basename(dout2), ".rds")
+
+    saveRDS(d, fout)
+
+    dout3 <- paste0(dout2, "csv")
+
+    if (!dir.exists(dout3)) {
+      dir.create(dout3)
+    }
+
+    # save each element of the list as a csv file
+    purrr::map(
+      names(d),
+      function(n) {
+        fout <- paste0(dout3, "/", n, ".csv")
+        readr::write_delim(d[[n]], fout, delim = ";")
+      }
+    )
+
+    # put all the csv files into a zip file
+    fout <- paste0(dout2, basename(dout2), ".zip")
+
+    finp <- list.files(dout3, pattern = "\\.csv", full.names = TRUE)
+
+    # use 7z + system to add all the csv files to a zip file
+    system(paste0("7z a -tzip -r ", fout, " ./", paste(finp, collapse = " ./")))
+
+    unlink(dout3, recursive = TRUE)
+
+    return(TRUE)
+  }
+)

pypkg/redatamlib/utils/utils.cpp

@@ -34,17 +34,13 @@ string ReplaceRootPath(const string &rootPath, const string &fileName) {
     normalizedRootPath.append("/");
   }
 
-  // Remove any leading "./" or ".\" from the file name
+  // Remove everything before the last '/' or '\'
   string normalizedFileName = fileName;
-  if (normalizedFileName.find("./") == 0 ||
-      normalizedFileName.find(".\\") == 0) {
-    normalizedFileName = normalizedFileName.substr(2);
+  size_t pos = normalizedFileName.find_last_of("/\\");
+  if (pos != string::npos) {
+    normalizedFileName = normalizedFileName.substr(pos + 1);
   }
 
-  size_t pos = normalizedFileName.find_first_not_of("/\\");
-  normalizedFileName =
-      (pos == string::npos) ? "" : normalizedFileName.substr(pos);
-
   string fullPath = normalizedRootPath + normalizedFileName;
   return fullPath;
 }

rpkg/src/redatamlib/utils/utils.cpp

@@ -33,17 +33,13 @@ string ReplaceRootPath(const string &rootPath, const string &fileName) {
     normalizedRootPath.append("/");
   }
 
-  // Remove any leading "./" or ".\" from the file name
+  // Remove everything before the last '/' or '\'
   string normalizedFileName = fileName;
-  if (normalizedFileName.find("./") == 0 ||
-      normalizedFileName.find(".\\") == 0) {
-    normalizedFileName = normalizedFileName.substr(2);
+  size_t pos = normalizedFileName.find_last_of("/\\");
+  if (pos != string::npos) {
+    normalizedFileName = normalizedFileName.substr(pos + 1);
   }
 
-  size_t pos = normalizedFileName.find_first_not_of("/\\");
-  normalizedFileName =
-      (pos == string::npos) ? "" : normalizedFileName.substr(pos);
-
   string fullPath = normalizedRootPath + normalizedFileName;
   return fullPath;
 }

src/utils/utils.cpp

@@ -40,17 +40,13 @@ string ReplaceRootPath(const string &rootPath, const string &fileName) {
     normalizedRootPath.append("/");
   }
 
-  // Remove any leading "./" or ".\" from the file name
+  // Remove everything before the last '/' or '\'
   string normalizedFileName = fileName;
-  if (normalizedFileName.find("./") == 0 ||
-      normalizedFileName.find(".\\") == 0) {
-    normalizedFileName = normalizedFileName.substr(2);
+  size_t pos = normalizedFileName.find_last_of("/\\");
+  if (pos != string::npos) {
+    normalizedFileName = normalizedFileName.substr(pos + 1);
   }
 
-  size_t pos = normalizedFileName.find_first_not_of("/\\");
-  normalizedFileName =
-      (pos == string::npos) ? "" : normalizedFileName.substr(pos);
-
   string fullPath = normalizedRootPath + normalizedFileName;
   return fullPath;
 }