Merge pull request #5 from long237/compressTesting

Adding Data source factory pattern. Fix some conflict in ignore file.

.gitignore

@@ -3,6 +3,10 @@ venv/
 HaskellCode/.vscode/tasks.json
 Notes/
 
+
+.vscode/*
+!.vscode/launch.json
+
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 ##
@@ -18,6 +22,9 @@ Notes/
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs
 
+# Mono auto generated files
+mono_crash.*
+
 # Build results
 [Dd]ebug/
 [Dd]ebugPublic/
@@ -25,6 +32,9 @@ Notes/
 [Rr]eleases/
 x64/
 x86/
+
+[Ww][Ii][Nn]32/
+
 [Aa][Rr][Mm]/
 [Aa][Rr][Mm]64/
 bld/
@@ -32,6 +42,9 @@ bld/
 [Oo]bj/
 [Ll]og/
 
+[Ll]ogs/
+
+
 # Visual Studio 2015/2017 cache/options directory
 .vs/
 # Uncomment if you have tasks that create the project's static files in wwwroot
@@ -44,9 +57,10 @@ Generated\ Files/
 [Tt]est[Rr]esult*/
 [Bb]uild[Ll]og.*
 
-# NUNIT
+# NUnit
 *.VisualState.xml
 TestResult.xml
+nunit-*.xml
 
 # Build Results of an ATL Project
 [Dd]ebugPS/
@@ -61,6 +75,9 @@ project.lock.json
 project.fragment.lock.json
 artifacts/
 
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
 # StyleCop
 StyleCopReport.xml
 
@@ -86,6 +103,10 @@ StyleCopReport.xml
 *.tmp_proj
 *_wpftmp.csproj
 *.log
+
+*.tlog
+
+
 *.vspscc
 *.vssscc
 .builds
@@ -140,6 +161,11 @@ _TeamCity*
 .axoCover/*
 !.axoCover/settings.json
 
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
 # Visual Studio code coverage results
 *.coverage
 *.coveragexml
@@ -187,6 +213,10 @@ PublishScripts/
 
 # NuGet Packages
 *.nupkg
+
+# NuGet Symbol Packages
+*.snupkg
+
 # The packages folder can be ignored because of Package Restore
 **/[Pp]ackages/*
 # except build/, which is used as an MSBuild target.
@@ -197,6 +227,9 @@ PublishScripts/
 *.nuget.props
 *.nuget.targets
 
+# Nuget personal access tokens and Credentials
+# nuget.config
+
 # Microsoft Azure Build Output
 csx/
 *.build.csdef
@@ -211,6 +244,9 @@ BundleArtifacts/
 Package.StoreAssociation.xml
 _pkginfo.txt
 *.appx
+*.appxbundle
+*.appxupload
+
 
 # Visual Studio cache files
 # files ending in .cache can be ignored
@@ -236,6 +272,7 @@ orleans.codegen.cs
 # Since there are multiple workflows, uncomment next line to ignore bower_components
 # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
 #bower_components/
+
 # ASP.NET Core default setup: bower directory is configured as wwwroot/lib/ and bower restore is true
 **/wwwroot/lib/
 
@@ -262,6 +299,10 @@ ServiceFabricBackup/
 *.bim.layout
 *.bim_*.settings
 *.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
 
 # Microsoft Fakes
 FakesAssemblies/
@@ -342,5 +383,41 @@ ASALocalRun/
 # Local History for Visual Studio
 .localhistory/
 
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+# VS Code files for those working on multiple tools
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+# Windows Installer files from build outputs
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# JetBrains Rider
+.idea/
+*.sln.iml
+
 # BeatPulse healthcheck temp database 
 healthchecksdb
+

src/Archive/.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            // Use IntelliSense to find out which attributes exist for C# debugging
+            // Use hover for the description of the existing attributes
+            // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
+            "name": ".NET Core Launch (console)",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceFolder}/app/bin/Debug/net6.0/app.dll",
+            "args": [],
+            "cwd": "${workspaceFolder}/app",
+            // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
+            "console": "internalConsole",
+            "stopAtEntry": false
+        },
+        {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach"
+        }
+    ]
+}

src/Archive/.vscode/tasks.json

@@ -0,0 +1,42 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "build",
+                "${workspaceFolder}/app/app.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "publish",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "publish",
+                "${workspaceFolder}/app/app.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "watch",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "watch",
+                "run",
+                "${workspaceFolder}/app/app.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        }
+    ]
+}

src/Archive/app/ArchiveController.cs

@@ -0,0 +1,51 @@
+using System;
+
+namespace app
+{
+    class ArchiveController{
+        static void Main(string[] args)
+        {
+            string dsName = "SQL";
+            string cmd = "";
+            Console.WriteLine("Hello World!");    
+
+            //SQLConnection myconnection = new SQLConnection();
+
+            // Creating a DataSource Factory to make Data sources
+            DataSourceFactory dsFactory = new DataSourceFactory();
+
+            // Creating a SQL data source objects to gain access to the SQL database
+            IDataSource dataSource = dsFactory.getDataSource("SQL");
+
+            // Read from the database
+            dataSource.ReadData("a");
+
+            // Write to the database
+            dataSource.WriteData("Insert newName");
+
+            // Update the datasource
+            dataSource.UpdateData();
+
+            // Delete from the datasource
+            dataSource.DeleteData();
+
+            Console.WriteLine("-----------------");
+            // Creating a Data Source for text files
+            IDataSource txtSource = dsFactory.getDataSource("txt");
+
+            // Read from the database
+            txtSource.ReadData("a");
+
+            // Write to the database
+            txtSource.WriteData("a");
+
+            // Update the datasource
+            txtSource.UpdateData();
+
+            // Delete from the datasource
+            txtSource.DeleteData();
+
+        }
+    }
+
+}

src/Archive/app/DataSourceFactory.cs

@@ -0,0 +1,18 @@
+
+class DataSourceFactory{
+    public IDataSource getDataSource(string name){
+        
+        if (name == "SQL"){
+            return new SQLSource();
+        }
+        else if (String.Equals(name, "TXT", StringComparison.OrdinalIgnoreCase)){
+            return new TxtSource();
+        }
+        else if(name == "XML"){
+            return null;
+        }
+        else {
+            return null;
+        }
+    }
+}

src/Archive/app/Factory.cs

@@ -0,0 +1,6 @@
+
+abstract class Factory{
+
+    //Method to make a datasource object
+    public abstract IDataSource getDataSource(string name);
+}

src/Archive/app/IDataSource.cs

@@ -0,0 +1,15 @@
+
+interface IDataSource{
+
+    // Method for reading data, return 0 for sucessful operation
+    int ReadData(string cmd);
+
+    // Method for Writing data to a data source
+    int WriteData(string cmd);
+
+    //Method for deleteing data from a data source, 0 for successful
+    int DeleteData();
+
+    // Method for updating data from a data source, 0 for sucessful
+    int UpdateData();
+}

src/Archive/app/SQLSource.cs

@@ -0,0 +1,41 @@
+
+class SQLSource  : IDataSource{ 
+    //private SQLConnection conn;
+
+    // public SQLSource(){
+    //     conn = new SQLConnection("usernamePasword");
+    // }
+
+    public int ReadData(string cmd){
+        try{
+            // conn.open();
+            // conn.Execute();
+            Console.WriteLine("Access a SQL database");
+            Console.WriteLine("Select * from archive");
+
+            // while (myReader)
+                // Print to console
+            // conn.close();
+            return 0;
+        }
+        catch (Exception e){
+            Console.WriteLine(e.Message);
+            return -1;
+        }
+    }
+
+    public int WriteData(string cmd){
+        Console.WriteLine("Writing data to a SQL database");
+        return 0;
+    }
+
+    public int DeleteData() {
+        Console.WriteLine("Deleting data from a SQL database");
+        return 0;
+    }
+
+    public int UpdateData() {
+        Console.WriteLine("Update data to a SQL database");
+        return 0;
+    }
+}

src/Archive/app/TxtSource.cs

@@ -0,0 +1,29 @@
+
+class TxtSource : IDataSource{
+    private string fileName;
+    public int ReadData(string a){
+        try{
+            Console.WriteLine("Access a text file");
+            return 0;
+        }
+        catch (Exception e){
+            Console.WriteLine(e.Message);
+            return -1;
+        }
+    }
+
+    public int WriteData(string cmd){
+        Console.WriteLine("Writing data to a text file");
+        return 0;
+    }
+
+    public int DeleteData() {
+        Console.WriteLine("Deleting data from a text file");
+        return 0;
+    }
+
+    public int UpdateData() {
+        Console.WriteLine("Update data to a text file");
+        return 0;
+    }
+}

src/Archive/app/app.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

src/dbCode/Bigtable.txt

@@ -22,6 +22,20 @@ ALTER TABLE key_block_size_8 key_block_size=8 row_format=compressed;
 INSERT INTO key_block_size_8 SELECT * FROM big_table;
 commit;
 
+-- Create a new table that is the same but compressed 16KB block size
+CREATE TABLE key_block_size_16 LIKE big_table;
+ALTER TABLE key_block_size_16 key_block_size=16 row_format=compressed;
+-- Insert 3 millions row into the table to compressed
+INSERT INTO key_block_size_16 SELECT * FROM big_table;
+commit;
+
+-- Create a new table that is the same but compressed 4KB block size
+CREATE TABLE key_block_size_4 LIKE big_table;
+ALTER TABLE key_block_size_4 key_block_size=4 row_format=compressed;
+-- Insert 3 millions row into the table to compressed
+INSERT INTO key_block_size_4 SELECT * FROM big_table;
+commit;
+
 drop table key_block_size_4;
 
 SELECT * FROM information_schema.INNODB_CMP;
@@ -29,4 +43,8 @@ SELECT * FROM information_schema.INNODB_CMPMEM;
 
 
 select count(*) as count_row from big_table;
-select count(*) as count_row from key_block_size_8;
+select count(*) as count_row from key_block_size_8;
+
+-- For the console
+-- SELECT * FROM information_schema.INNODB_CMPMEM\G
+-- SELECT * FROM information_schema.INNODB_CMP\G

src/dbCode/Performance.txt

@@ -0,0 +1,7 @@
+Number of Records: 3,318,784 
+Uncompressed size: 683.00 MB
+Block Size	Time			Space(MB)
+2KB		3m 47s 207ms 		181.63
+4KB		2m 42s	961ms		159.50
+8KB		57s 572ms		342.50
+16KB