feature/add-ability-to-perform-partail-work

Add ability to perform partail work

Author: Arpan-Kreeti

.formatter.exs

@@ -28,6 +28,11 @@ csv2sql-*.tar
 # schema file
 schema.sql
 
-
 # config file
 /config.env
+
+# Formatting file
+.formatter.exs
+
+# escipt binary
+csv2sql

.gitignore

@@ -43,7 +43,7 @@ Csv2 sql automatically...
 
 Download the [csv2sql executable](https://github.com/Arpan-Kreeti/csv2sql/blob/master/executable/csv2sql.zip).
 
-Extract the zip and follow the instructions in the ```readme.md``` file
+Extract the zip and follow the instructions in the ```read me.md``` file
 
 ### From repository
 
@@ -55,13 +55,21 @@ Once that is done, source the environment variables by ```source ./config.env.sa
 
 Then start the application by ```iex -S mix```
 
+Execute the followinf function to begin the process ```Csv2sql.main(nil)```
+
 Thats all !
 
+### Troubleshooting
 
-## Further improvements or features that might come...
+In case you face datatrucation errors or other issues when inserting data, delete the database in which
+csv2sql inserts data, so that it will be recreated with correct encoding and collation next time.
+By default csv2sql uses ```utf8mb4``` character set and ```utf8mb4_general_ci``` collation when
+creating a database.
 
-* Make a standalone executable of the application
+If you face database connection timeout errors try reducing the worker and db_worker count in the configurations
+or change the database timeout, pool size and other related database configurations.
 
-* Support other popular databases like postgres, mssql server, etc
 
-* Allow users to perform partial tasks only like creating schema, inserting data or validating data.
+## Further improvements or features to expect...
+
+* Support other popular databases like postgres, mssql server, etc

README.md

@@ -16,6 +16,18 @@ export csv2sql_imported_csv_directory=?
 # The directory were the csvs will be moved after they are validated, make sure it is present and is empty
 export csv2sql_validated_csv_directory=?
 
+# Set whether you want to make schema file
+export csv2sql_set_make_schema="true"
+
+# Set whether you want to insert the schema file that is generated (this option will only work if set_make_schema="true")
+export csv2sql_set_insert_schema="true"
+
+# Set whether to insert the CSVs into the database
+export csv2sql_set_insert_data="true"
+
+# Set whether to validate if the csvs have been inserted correctly
+export csv2sql_set_validate="true"
+
 # mysql username
 export csv2sql_username=?
 

config.env.sample

@@ -1,5 +1,6 @@
 defmodule Csv2sql do
-  def main(args \\ []) do
+  def main(_args \\ []) do
+    Csv2sql.Helpers.greet()
     # Load configuration varaibles dynamically for escripts, this is required
     # since configuration variables are set to whatever they where when the
     # escript was build and cannot be changed later
@@ -29,7 +30,18 @@ defmodule Csv2sql do
            db_worker_count: System.get_env("csv2sql_db_worker_count") |> String.to_integer(),
            source_csv_directory: System.get_env("csv2sql_source_csv_directory"),
            imported_csv_directory: System.get_env("csv2sql_imported_csv_directory"),
-           validated_csv_directory: System.get_env("csv2sql_validated_csv_directory")
+           validated_csv_directory: System.get_env("csv2sql_validated_csv_directory"),
+           set_validate:
+             if(System.get_env("csv2sql_set_validate") == "true", do: true, else: false)
+         ]},
+        {Csv2sql.Worker,
+         [
+           set_make_schema:
+             if(System.get_env("csv2sql_set_make_schema") == "true", do: true, else: false),
+           set_insert_schema:
+             if(System.get_env("csv2sql_set_insert_schema") == "true", do: true, else: false),
+           set_insert_data:
+             if(System.get_env("csv2sql_set_insert_data") == "true", do: true, else: false)
          ]},
         {Csv2sql.Repo,
          [

config/config.exs

@@ -2,15 +2,29 @@ defmodule Csv2sql.Application do
   use Application
 
   def start(_type, _args) do
-    children = [
-      {Csv2sql.TimerServer, Time.utc_now()},
-      Csv2sql.Repo,
-      Csv2sql.FileServer,
-      Csv2sql.JobQueueServer,
-      Csv2sql.DbWorkerSupervisor,
-      Csv2sql.WorkerSupervisor,
-      Csv2sql.MainServer
-    ]
+    set_validate = Application.get_env(:csv2sql, Csv2sql.MainServer)[:set_validate]
+    set_insert_schema = Application.get_env(:csv2sql, Csv2sql.Worker)[:set_insert_schema]
+    set_insert_data = Application.get_env(:csv2sql, Csv2sql.Worker)[:set_insert_data]
+
+    repo_supervisor =
+      if set_validate || set_insert_schema || set_insert_data do
+        [Csv2sql.Repo]
+      else
+        []
+      end
+
+    children =
+      [
+        {Csv2sql.TimerServer, Time.utc_now()}
+      ]
+      |> Kernel.++(repo_supervisor)
+      |> Kernel.++([
+        Csv2sql.FileServer,
+        Csv2sql.JobQueueServer,
+        Csv2sql.DbWorkerSupervisor,
+        Csv2sql.WorkerSupervisor,
+        Csv2sql.MainServer
+      ])
 
     opts = [strategy: :one_for_one, name: Csv2sql.Supervisor]
     Supervisor.start_link(children, opts)

executable/csv2sql.zip

@@ -52,7 +52,7 @@ defmodule Csv2sql.Database do
 
     Ecto.Adapters.SQL.query!(
       Repo,
-      "SET GLOBAL SQL_MODE=\"NO_BACKSLASH_ESCAPES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_ZERO_IN_DATE\";",
+      "SET GLOBAL SQL_MODE=\"NO_BACKSLASH_ESCAPES,NO_ENGINE_SUBSTITUTION,NO_ZERO_IN_DATE\";",
       []
     )
   end

lib/csv2sql.ex

@@ -21,14 +21,19 @@ defmodule Csv2sql.MainServer do
   end
 
   def handle_info(:kickoff, worker_count) do
-    Csv2sql.Helpers.greet()
+    set_insert_schema = Application.get_env(:csv2sql, Csv2sql.Worker)[:set_insert_schema]
+    set_insert_data = Application.get_env(:csv2sql, Csv2sql.Worker)[:set_insert_data]
 
-    Csv2sql.Database.prepare_db()
+    if set_insert_schema || set_insert_data do
+      Csv2sql.Database.prepare_db()
+    end
 
     schema_file_path = Application.get_env(:csv2sql, Csv2sql.MainServer)[:db_worker_count]
     db_worker_count = Application.get_env(:csv2sql, Csv2sql.MainServer)[:db_worker_count]
 
-    File.rm("#{schema_file_path}/schema.sql")
+    if Application.get_env(:csv2sql, Csv2sql.Worker)[:set_make_schema] do
+      File.rm("#{schema_file_path}/schema.sql")
+    end
 
     1..worker_count
     |> Enum.each(fn _ -> Csv2sql.WorkerSupervisor.add_worker() end)
@@ -40,6 +45,8 @@ defmodule Csv2sql.MainServer do
   end
 
   def handle_cast(:done, 1) do
+    set_validate = Application.get_env(:csv2sql, Csv2sql.MainServer)[:set_validate]
+
     wait_for_pending_jobs()
 
     :timer.sleep(2000)
@@ -65,12 +72,16 @@ defmodule Csv2sql.MainServer do
       end
     )
 
-    Csv2sql.Helpers.print_msg("\nValidation Process Started...\n\n", :green)
+    if(set_validate) do
+      Csv2sql.Helpers.print_msg("\nValidation Process Started...\n\n", :green)
 
-    imported_csv_directory =
-      Application.get_env(:csv2sql, Csv2sql.MainServer)[:imported_csv_directory]
+      imported_csv_directory =
+        Application.get_env(:csv2sql, Csv2sql.MainServer)[:imported_csv_directory]
 
-    Csv2sql.ImportValidator.validate_import(imported_csv_directory)
+      Csv2sql.ImportValidator.validate_import(imported_csv_directory)
+    else
+      Csv2sql.Helpers.print_msg("\nValidation Process Skipped...\n\n", :green)
+    end
 
     Csv2sql.TimerServer.get_time_spend()