diff --git a/doc/Business Docs/Cloud_Service_Tech_Approval.pdf b/doc/Business Docs/Cloud_Service_Tech_Approval.pdf new file mode 100644 index 0000000..2614444 Binary files /dev/null and b/doc/Business Docs/Cloud_Service_Tech_Approval.pdf differ diff --git a/doc/Business Docs/Project Roadmap.drawio.png b/doc/Business Docs/Project Roadmap.drawio.png index beac9db..3bd8dfd 100644 Binary files a/doc/Business Docs/Project Roadmap.drawio.png and b/doc/Business Docs/Project Roadmap.drawio.png differ diff --git a/doc/Images/references.txt b/doc/Images/references.txt new file mode 100644 index 0000000..acd8227 --- /dev/null +++ b/doc/Images/references.txt @@ -0,0 +1,23 @@ +https://aws.amazon.com/rds/ +https://aws.amazon.com/rds/features/#administration +https://docs.microsoft.com/en-us/azure/mariadb/concepts-supported-versions +https://cloud.google.com/sql/pricing +https://calculator.aws/#/createCalculator/RDSMariaDB +https://azure.microsoft.com/en-us/pricing/calculator/ +https://aws.amazon.com/about-aws/whats-new/2019/06/rds-storage-auto-scaling/ +https://aws.amazon.com/blogs/database/scaling-your-amazon-rds-instance-vertically-and-horizontally/ +https://cloud.google.com/architecture/elastically-scaling-your-mysql-environment +https://techcommunity.microsoft.com/t5/azure-database-support-blog/how-to-auto-scale-azure-sql-databases/ba-p/2235441 +https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html +https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html +https://docs.microsoft.com/en-us/azure/security/fundamentals/encryption-overview +https://cloud.google.com/security/encryption/default-encryption +https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Overview.Encryption.html +https://cloud.google.com/compute/docs/regions-zones +https://docs.microsoft.com/en-us/azure/availability-zones/az-overview +https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html +https://azure.microsoft.com/en-us/pricing/details/bandwidth/ +https://aws.amazon.com/ec2/pricing/on-demand/ +https://cloud.google.com/vpc/network-pricing +https://azure.microsoft.com/en-us/pricing/details/azure-sql-database/single/ +https://azure.microsoft.com/en-us/pricing/details/azure-sql-managed-instance/single/ diff --git a/doc/LowLevel/archiving.pdf b/doc/LowLevel/archiving.pdf new file mode 100644 index 0000000..98d4a79 Binary files /dev/null and b/doc/LowLevel/archiving.pdf differ diff --git a/doc/LowLevel/archiving.txt b/doc/LowLevel/archiving.txt new file mode 100644 index 0000000..2bc89f6 --- /dev/null +++ b/doc/LowLevel/archiving.txt @@ -0,0 +1,49 @@ +title Use Case: Archiving + +participant ArchiveController.cs +participant DataConnection +participant DataSourceFactory +participant SQLSource +participant MariaDB + +activate ArchiveController.cs +activate DataConnection +ArchiveController.cs ->DataConnection:static DataConnection getDataConnection() +DataConnection -->ArchiveController.cs:return DataConnection +deactivate DataConnection + +activate DataSourceFactory +DataSourceFactory ->DataSourceFactory:DataSourceFactory()\nconstructor +ArchiveController.cs <-- DataSourceFactory:return DataSourceFactory +deactivate DataSourceFactory + +activate SQLSource +ArchiveController.cs->SQLSource: IDataSource DataSourceFact.GetDataSource(String sourceName) +SQLSource->SQLSource: SQLSource()\nconstructor +SQLSource --> ArchiveController.cs :return SQLSource + +loop #lightblue while true +alt #lightgreen date == 1 +ArchiveController.cs -> SQLSource: int Compress(DataConnection conn, IDataSource ds, String logList[]) +activate MariaDB +alt #lightgreen archive == 0 +SQLSource -> MariaDB: int createArchive(DataConnection conn, String SQLcmd) +MariaDB -->SQLSource:return 0 +end + +SQLSource ->SQLSource: int updateArchive(DataConnection conn, String logList[]) +SQLSource ->MariaDB: int CopremovyEntries(DataConnection conn, String logList[]) +MariaDB --> SQLSource: return 0 + +SQLSource ->MariaDB: int RemoveEntries(DataConnection conn, String logList[]) +MariaDB-->SQLSource:return 0 +end + + +deactivate MariaDB + +SQLSource-->ArchiveController.cs:return 0 +end + +deactivate ArchiveController.cs +deactivate SQLSource \ No newline at end of file diff --git a/doc/LowLevel/archiving2.png b/doc/LowLevel/archiving2.png new file mode 100644 index 0000000..b9ccdd9 Binary files /dev/null and b/doc/LowLevel/archiving2.png differ diff --git a/doc/LowLevel/archivingWithError.pdf b/doc/LowLevel/archivingWithError.pdf new file mode 100644 index 0000000..8c6386b Binary files /dev/null and b/doc/LowLevel/archivingWithError.pdf differ diff --git a/doc/LowLevel/archivingWithError2.pdf b/doc/LowLevel/archivingWithError2.pdf new file mode 100644 index 0000000..363b9bd Binary files /dev/null and b/doc/LowLevel/archivingWithError2.pdf differ diff --git a/doc/LowLevel/archiving_5.txt b/doc/LowLevel/archiving_5.txt new file mode 100644 index 0000000..a4f04ab --- /dev/null +++ b/doc/LowLevel/archiving_5.txt @@ -0,0 +1,44 @@ +title Use Case: Archiving + +participant ArchiveController.cs +participant DataSourceFactory +participant SQLSource +participant MariaDB + +activate ArchiveController.cs +ArchiveController.cs->DataSourceFactory:new DataSourceFactory() +activate DataSourceFactory +DataSourceFactory ->DataSourceFactory:DataSourceFactory()\nconstructor +ArchiveController.cs <-- DataSourceFactory:return DataSourceFactory +deactivate DataSourceFactory + +activate SQLSource +ArchiveController.cs->SQLSource: IDataSource DataSourceFact.GetDataSource(String sourceName) +SQLSource->SQLSource: SQLSource()\nconstructor +SQLSource --> ArchiveController.cs :return SQLSource + +loop #lightblue while true +alt #lightgreen date == 1 +ArchiveController.cs -> SQLSource: int Compress(DataConnection conn, IDataSource ds, String logList[]) +activate MariaDB +alt #lightgreen archive == 0 +SQLSource -> MariaDB: int createArchive(DataConnection conn, String SQLcmd) +MariaDB -->SQLSource:return 0 +end + +SQLSource ->SQLSource: int updateArchive(DataConnection conn, String logList[]) +SQLSource ->MariaDB: int CopremovyEntries(DataConnection conn, String logList[]) +MariaDB --> SQLSource: return 0 + +SQLSource ->MariaDB: int RemoveEntries(DataConnection conn, String logList[]) +MariaDB-->SQLSource:return 0 +end + + +deactivate MariaDB + +SQLSource-->ArchiveController.cs:return 0 +end + +deactivate ArchiveController.cs +deactivate SQLSource \ No newline at end of file diff --git a/doc/LowLevel/archivngWithError.txt b/doc/LowLevel/archivngWithError.txt new file mode 100644 index 0000000..f58633c --- /dev/null +++ b/doc/LowLevel/archivngWithError.txt @@ -0,0 +1,63 @@ +title Use Case: Archiving + +participant ArchiveController.cs +participant DataConnection +participant DataSourceFactory +participant SQLSource +participant MariaDB + +activate ArchiveController.cs +ArchiveController.cs ->DataConnection:static DataConnection getDataConnection() +DataConnection -->ArchiveController.cs:return DataConnection + +ArchiveController.cs->DataSourceFactory:new DataSourceFactory() +activate DataSourceFactory +DataSourceFactory ->DataSourceFactory:DataSourceFactory()\nconstructor +ArchiveController.cs <-- DataSourceFactory:return DataSourceFactory +deactivate DataSourceFactory + +activate SQLSource +ArchiveController.cs->SQLSource: IDataSource DataSourceFact.GetDataSource(String sourceName) +SQLSource->SQLSource: SQLSource()\nconstructor +SQLSource --> ArchiveController.cs :return SQLSource + +loop #lightblue while true +alt #lightgreen date == 1 +ArchiveController.cs -> SQLSource: int Compress(DataConnection conn, IDataSource ds, String logList[]) +activate MariaDB +alt #lightgreen archive == 0 (If Archive have not exist already +SQLSource -> MariaDB: int createArchive(DataConnection conn, String SQLcmd) +SQLSource <-- MariaDB:return 0 +end + +alt #red No error when doing executing copy SQL command +SQLSource ->SQLSource: int updateArchive(DataConnection conn, String logList[]) +SQLSource ->MariaDB: int CopremovyEntries(DataConnection conn, String logList[]) +MariaDB --> SQLSource: return 0 +SQLSource -->ArchiveController.cs: return 0 +else Error occur when executing copy SQL command on Database +MariaDB -->SQLSource: throw SQLException +SQLSource -->ArchiveController.cs: throw SQLException +ArchiveController.cs ->ArchiveController.cs: Console.WriteLine(SQLException.Message) \n (As a form of handling error) +end + +alt #red No error when doing executing remove SQL command +SQLSource ->MariaDB: int RemoveEntries(DataConnection conn, String logList[]) +MariaDB-->SQLSource:return 0 +SQLSource -->ArchiveController.cs: return 0 + +else Error occur when executing removing SQL command on Database +MariaDB -->SQLSource: throw SQLException +SQLSource -->ArchiveController.cs: throw SQLException +ArchiveController.cs ->ArchiveController.cs: Console.WriteLine(SQLException.Message) \n (As a form of handling error) +end +end + + +deactivate MariaDB + +SQLSource-->ArchiveController.cs:return 0 +end + +deactivate ArchiveController.cs +deactivate SQLSource \ No newline at end of file diff --git a/doc/Resources/1565-Article Text-4599-1-10-20200320.pdf b/doc/Resources/1565-Article Text-4599-1-10-20200320.pdf new file mode 100644 index 0000000..5092940 Binary files /dev/null and b/doc/Resources/1565-Article Text-4599-1-10-20200320.pdf differ diff --git a/doc/Resources/BENCHMARKING_POPULAR_OPEN_SOURCE_RDBMS_A1.pdf b/doc/Resources/BENCHMARKING_POPULAR_OPEN_SOURCE_RDBMS_A1.pdf new file mode 100644 index 0000000..a895ead Binary files /dev/null and b/doc/Resources/BENCHMARKING_POPULAR_OPEN_SOURCE_RDBMS_A1.pdf differ diff --git a/doc/Resources/Experiments of Search Query Performance for SQL-Based Open Source Databases.pdf b/doc/Resources/Experiments of Search Query Performance for SQL-Based Open Source Databases.pdf new file mode 100644 index 0000000..f7951b1 Binary files /dev/null and b/doc/Resources/Experiments of Search Query Performance for SQL-Based Open Source Databases.pdf differ diff --git a/doc/Resources/Graph/insert.png b/doc/Resources/Graph/insert.png new file mode 100644 index 0000000..f60a473 Binary files /dev/null and b/doc/Resources/Graph/insert.png differ diff --git a/doc/Resources/Graph/joinQuery.png b/doc/Resources/Graph/joinQuery.png new file mode 100644 index 0000000..2b45fd9 Binary files /dev/null and b/doc/Resources/Graph/joinQuery.png differ diff --git a/doc/Resources/Graph/multiquery.png b/doc/Resources/Graph/multiquery.png new file mode 100644 index 0000000..d33f794 Binary files /dev/null and b/doc/Resources/Graph/multiquery.png differ diff --git a/doc/Resources/Graph/rangeQuery.png b/doc/Resources/Graph/rangeQuery.png new file mode 100644 index 0000000..212aaf1 Binary files /dev/null and b/doc/Resources/Graph/rangeQuery.png differ diff --git a/doc/Resources/Graph/rangeTable.png b/doc/Resources/Graph/rangeTable.png new file mode 100644 index 0000000..80fa919 Binary files /dev/null and b/doc/Resources/Graph/rangeTable.png differ diff --git a/doc/Resources/Graph/read.png b/doc/Resources/Graph/read.png new file mode 100644 index 0000000..1434829 Binary files /dev/null and b/doc/Resources/Graph/read.png differ diff --git a/doc/Resources/Graph/singleQuery.png b/doc/Resources/Graph/singleQuery.png new file mode 100644 index 0000000..06a6bd8 Binary files /dev/null and b/doc/Resources/Graph/singleQuery.png differ diff --git a/doc/Resources/Graph/update.png b/doc/Resources/Graph/update.png new file mode 100644 index 0000000..19d92f5 Binary files /dev/null and b/doc/Resources/Graph/update.png differ diff --git a/doc/Resources/Hairah_2021_J._Phys.__Conf._Ser._1844_012021.pdf b/doc/Resources/Hairah_2021_J._Phys.__Conf._Ser._1844_012021.pdf new file mode 100644 index 0000000..d15e0dd Binary files /dev/null and b/doc/Resources/Hairah_2021_J._Phys.__Conf._Ser._1844_012021.pdf differ diff --git a/doc/Resources/IJ0350020.pdf b/doc/Resources/IJ0350020.pdf new file mode 100644 index 0000000..8f92c59 Binary files /dev/null and b/doc/Resources/IJ0350020.pdf differ diff --git a/doc/Resources/dbmsPapers.txt b/doc/Resources/dbmsPapers.txt new file mode 100644 index 0000000..6420a58 --- /dev/null +++ b/doc/Resources/dbmsPapers.txt @@ -0,0 +1,2 @@ +https://csu-lb.primo.exlibrisgroup.com/permalink/01CALS_ULB/dfriqe/cdi_proquest_journals_2512916339 +https://csu-lb.primo.exlibrisgroup.com/permalink/01CALS_ULB/dfriqe/cdi_nrf_kci_oai_kci_go_kr_ARTI_4401754 \ No newline at end of file diff --git a/src/dbCode/Bigtable.txt b/src/dbCode/Bigtable.txt new file mode 100644 index 0000000..16aaf2a --- /dev/null +++ b/src/dbCode/Bigtable.txt @@ -0,0 +1,32 @@ +-- Create an uncompressed table with a million or two rows. (3 millions row) +CREATE TABLE big_table AS SELECT * FROM information_schema.columns; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; +INSERT INTO big_table SELECT * FROM big_table; + +-- How many rows does this has? +SELECT count(*) FROM information_schema.columns; + +-- Create a new table that is the same but compressed +CREATE TABLE key_block_size_8 LIKE big_table; +ALTER TABLE key_block_size_8 key_block_size=8 row_format=compressed; + +-- Insert 3 millions row into the table to compressed +INSERT INTO key_block_size_8 SELECT * FROM big_table; +commit; + +drop table key_block_size_4; + +SELECT * FROM information_schema.INNODB_CMP; +SELECT * FROM information_schema.INNODB_CMPMEM; + + +select count(*) as count_row from big_table; +select count(*) as count_row from key_block_size_8; \ No newline at end of file diff --git a/src/dbCode/archiving.sql b/src/dbCode/archiving.sql new file mode 100644 index 0000000..8eabdf8 --- /dev/null +++ b/src/dbCode/archiving.sql @@ -0,0 +1,57 @@ +create table archive +( + LtimeStamp datetime null, + logID int not null + primary key, + LvName varchar(50) not null, + catName varchar(50) not null, + userOP varchar(50) not null, + logMessage varchar(255) not null +); + +create table logcategories +( + catName varchar(50) not null + primary key, + catComment varchar(255) not null +); + +create table loglevel +( + LvName varchar(50) not null + primary key, + LvComment varchar(500) not null +); + +create table log +( + LtimeStamp datetime default current_timestamp() null, + logID int auto_increment + primary key, + LvName varchar(50) not null, + catName varchar(50) not null, + userOP varchar(50) not null, + logMessage varchar(255) not null, + constraint LogCatName_fk + foreign key (catName) references logcategories (catName), + constraint LogLvName_fk + foreign key (LvName) references loglevel (LvName) +); + +create table persons +( + PersonID int null, + LastName varchar(255) null, + FirstName varchar(255) null, + Address varchar(255) null, + City varchar(255) null +); + +create table users +( + user_id int not null + primary key, + b varchar(200) null +); + + diff --git a/src/dbCode/archivingDDL.txt b/src/dbCode/archivingDDL.txt new file mode 100644 index 0000000..2e9a684 --- /dev/null +++ b/src/dbCode/archivingDDL.txt @@ -0,0 +1,103 @@ +create schema Hobby; + +SET Global innodb_file_per_table=ON; +-- SET GLOBAL innodb_file_format='Barracuda'; + +SET GLOBAL innodb_default_row_format='dynamic'; + +SET GLOBAL innodb_compression_algorithm='zlib'; + +-- Check for system settings +SHOW VARIABLES LIKE 'innodb_compression_algorithm'; +SHOW VARIABLES LIKE 'innodb_default_row_format'; +SHOW VARIABLES LIKE '%innodb%'; + +CREATE TABLE Persons ( + PersonID int, + LastName varchar(255), + FirstName varchar(255), + Address varchar(255), + City varchar(255) +); + +CREATE DATABASE testDB; + +-- Hobby DDL +-- Log level table +create table LogLevel +( + LvName varchar(50) not null, + LvComment varchar(500) not null, + constraint LogLevel_pk + primary key (LvName) +); + +-- Log categories table +create table LogCategories +( + catName varchar(50) not null, + catComment varchar(255) not null, + constraint LogCategories_pk primary key (catName) +); + +-- DDL for log table +create table Log +( + LtimeStamp datetime default CURRENT_TIMESTAMP null, + logID int auto_increment not null, + LvName varchar(50) not null, + catName varchar(50) not null, + userOP varchar(50) not null, + logMessage varchar(255) not null, + + constraint LogLvName_fk foreign key (LvName) references loglevel (LvName), + constraint LogCatName_fk foreign key (catName) references logcategories (catName), + constraint Log_pk primary key (logID) + +); + +create table Archive +( + LtimeStamp datetime null, + logID int not null, + LvName varchar(50) not null, + catName varchar(50) not null, + userOP varchar(50) not null, + logMessage varchar(255) not null, + + constraint Archive_pk primary key (logID) +) + ENGINE=InnoDB + PAGE_COMPRESSED=1 + PAGE_COMPRESSION_LEVEL=9; + +drop table archive; + +-- Dummy data for testing purposes +-- Log categories data +INSERT into logcategories(catName, catComment) values + ('View', 'view layer'), + ('Business', 'business layer'), + ('Server', 'server layer'), + ('Data', 'data layer'); + +-- Log level data +INSERT into loglevel(lvname, lvcomment) values + ('Info', 'some sys flow'), + ('Debug', 'info for fixing bugs'), + ('Warning', 'track system failure'), + ('Error', 'some sys errors'); + +-- Log table data +INSERT into log(lvname, catname, userop, logmessage) values + ('Info', 'View', 'create some projects', 'new account created'), + ('Info', 'Business', 'create some projects', 'new projects made'), + ('Info', 'View', 'log out', 'log out successful'), + ('Info', 'Business', 'log in', 'log in successfully'), + ('Info', 'View', 'search for projects', 'result return'); + +select * from logcategories; +select * from log; + + + diff --git a/src/dbCode/arhiving.sql b/src/dbCode/arhiving.sql new file mode 100644 index 0000000..2b5d61d --- /dev/null +++ b/src/dbCode/arhiving.sql @@ -0,0 +1,39 @@ +create table logcategories +( + catName varchar(50) not null + primary key, + catComment varchar(255) not null +); + +create table loglevel +( + LvName varchar(50) not null + primary key, + LvComment varchar(500) not null +); + +create table log +( + LtimeStamp datetime default current_timestamp() null, + logID int auto_increment + primary key, + LvName varchar(50) not null, + catName varchar(50) not null, + userOP varchar(50) not null, + logMessage varchar(255) not null, + constraint LogCatName_fk + foreign key (catName) references logcategories (catName), + constraint LogLvName_fk + foreign key (LvName) references loglevel (LvName) +); + +create table persons +( + PersonID int null, + LastName varchar(255) null, + FirstName varchar(255) null, + Address varchar(255) null, + City varchar(255) null +); + +