From 6ddad76cf789c547ad77dc5c4803102db7e5cec7 Mon Sep 17 00:00:00 2001 From: Vidar Andresen Date: Wed, 23 Aug 2023 16:37:24 -0500 Subject: [PATCH] SQLite indexer Implemented support for building index in sqlite for the serverless GUI --- .../Pages/DataIndex/CreateIndex.razor | 20 ++++++++++--- .../Services/DataIndexerServerLess.cs | 3 +- DatabaseManager.Services.IndexSqlite/Api.cs | 10 +++++-- .../Helpers/IndexManagement.cs | 28 +++++++++--------- .../Models/BuildIndexParameters.cs | 2 +- .../Services/IndexAccess.cs | 2 +- .../mydatabase.db | Bin 11239424 -> 11239424 bytes 7 files changed, 41 insertions(+), 24 deletions(-) diff --git a/DatabaseManager.BlazorComponents/Pages/DataIndex/CreateIndex.razor b/DatabaseManager.BlazorComponents/Pages/DataIndex/CreateIndex.razor index 3b59e19..61172b4 100644 --- a/DatabaseManager.BlazorComponents/Pages/DataIndex/CreateIndex.razor +++ b/DatabaseManager.BlazorComponents/Pages/DataIndex/CreateIndex.razor @@ -49,7 +49,7 @@ else
-

Target Database: @singleton.TargetConnector

+

Target Database: @target

@@ -65,6 +65,7 @@ else private List options { get; set; } = new List(); private string source { get; set; } = "Choose Connector"; public string SqlText { get; set; } + public string target { get; set; } List taxonomies; CreateIndexParameters iParameters = new CreateIndexParameters(); private string statusMessage = "Not started"; @@ -76,14 +77,25 @@ else { if (string.IsNullOrEmpty(singleton.TargetConnector)) { - await displayMessage.DisplayErrorMessage("Please select a data connector"); - navigationManager.NavigateTo("/"); + if (SD.Sqlite) + { + target = "SQLite"; + } + else + { + await displayMessage.DisplayErrorMessage("Please select a data connector"); + navigationManager.NavigateTo("/"); + } + } + else + { + target = singleton.TargetConnector; } List fileList = await createIndex.GetTaxonomies(); var files = fileList.Select(s => s.Name); taxonomies = files.ToList(); - source = singleton.TargetConnector; + if (SD.Sqlite == false) source = singleton.TargetConnector; connectParameters = await dataSources.GetSources(); foreach (ConnectParameters conn in connectParameters) diff --git a/DatabaseManager.BlazorComponents/Services/DataIndexerServerLess.cs b/DatabaseManager.BlazorComponents/Services/DataIndexerServerLess.cs index a9a3380..926ab4a 100644 --- a/DatabaseManager.BlazorComponents/Services/DataIndexerServerLess.cs +++ b/DatabaseManager.BlazorComponents/Services/DataIndexerServerLess.cs @@ -26,7 +26,8 @@ public DataIndexerServerLess(IHttpClientFactory clientFactory, SingletonServices public async Task Create(CreateIndexParameters iParameters) { - if (string.IsNullOrEmpty(SD.IndexAPIBase)) url = $"api/createindex"; + if (SD.Sqlite == true) url = SD.IndexAPIBase.BuildFunctionUrl("/BuildIndex", $"", SD.IndexKey); + else if (string.IsNullOrEmpty(SD.IndexAPIBase)) url = $"api/createindex"; else url = SD.IndexAPIBase.BuildFunctionUrl("/api/BuildIndex", $"", SD.IndexKey); Console.WriteLine(url); ResponseDto response = await this.SendAsync(new ApiRequest() diff --git a/DatabaseManager.Services.IndexSqlite/Api.cs b/DatabaseManager.Services.IndexSqlite/Api.cs index 8774929..2228e23 100644 --- a/DatabaseManager.Services.IndexSqlite/Api.cs +++ b/DatabaseManager.Services.IndexSqlite/Api.cs @@ -95,16 +95,20 @@ private static async Task CreateIndexDatabase(IIndexAccess idxAccess) private static async Task BuildIndex(BuildIndexParameters idxParms, IIndexAccess idxAccess, IDataSourceService dataSource) { + ResponseDto response = new(); try { - if (string.IsNullOrEmpty(idxParms.TaxonomyFile)) return Results.BadRequest("Taxonomy file is missing"); + if (string.IsNullOrEmpty(idxParms.Taxonomy)) return Results.BadRequest("Taxonomy file is missing"); await idxAccess.BuildIndex(idxParms); - return Results.Ok(); + response.IsSuccess = true; } catch (Exception ex) { - return Results.Problem(ex.Message); + response.IsSuccess = false; + string newString = $"BuildIndex: Could not build index, {ex}"; + response.ErrorMessages.Insert(0, newString); } + return Results.Ok(response); } private static async Task GetDmIndexes(int? id, IIndexAccess idxAccess) diff --git a/DatabaseManager.Services.IndexSqlite/Helpers/IndexManagement.cs b/DatabaseManager.Services.IndexSqlite/Helpers/IndexManagement.cs index f08e54d..2b840ed 100644 --- a/DatabaseManager.Services.IndexSqlite/Helpers/IndexManagement.cs +++ b/DatabaseManager.Services.IndexSqlite/Helpers/IndexManagement.cs @@ -5,16 +5,16 @@ namespace DatabaseManager.Services.IndexSqlite.Helpers { public class IndexManagement { - private readonly string azureConnectionString; + //private readonly string azureConnectionString; //private readonly IFileStorageServiceCommon _fileStorage; - private readonly string taxonomyShare = "taxonomy"; + //private readonly string taxonomyShare = "taxonomy"; //private DbUtilities _dbConn; //private readonly DapperDataAccess _dp; //private readonly IIndexDBAccess _indexData; - public IndexManagement(string azureConnectionString) - { - this.azureConnectionString = azureConnectionString; + //public IndexManagement(string azureConnectionString) + //{ + //this.azureConnectionString = azureConnectionString; //var builder = new ConfigurationBuilder(); //IConfiguration configuration = builder.Build(); //_fileStorage = new AzureFileStorageServiceCommon(configuration); @@ -22,7 +22,7 @@ public IndexManagement(string azureConnectionString) //_dbConn = new DbUtilities(); //_dp = new DapperDataAccess(); //_indexData = new IndexDBAccess(_dp); - } + //} //public async Task GetTaxonomyFile(string taxonomyFile) @@ -126,13 +126,13 @@ public IndexManagement(string azureConnectionString) // return result; //} - public async Task CreateIndex(BuildIndexParameters indexParm) - { - if (string.IsNullOrEmpty(indexParm.TaxonomyFile)) - { - Exception error = new Exception($"Taxonomy not selected"); - throw error; - } + //public async Task CreateIndex(BuildIndexParameters indexParm) + //{ + //if (string.IsNullOrEmpty(indexParm.Taxonomy)) + //{ + // Exception error = new Exception($"Taxonomy not selected"); + // throw error; + //} //Sources sr = new Sources(azureConnectionString); //ConnectParameters target = await sr.GetSourceParameters(indexParm.TargetName); //ConnectParameters source = await sr.GetSourceParameters(indexParm.SourceName); @@ -148,7 +148,7 @@ public async Task CreateIndex(BuildIndexParameters indexParm) // } //} //index.CloseIndex(); - } + //} //private static IndexFileDefinition ProcessIndexFileTokens(JToken token) //{ diff --git a/DatabaseManager.Services.IndexSqlite/Models/BuildIndexParameters.cs b/DatabaseManager.Services.IndexSqlite/Models/BuildIndexParameters.cs index 8670d19..84e7228 100644 --- a/DatabaseManager.Services.IndexSqlite/Models/BuildIndexParameters.cs +++ b/DatabaseManager.Services.IndexSqlite/Models/BuildIndexParameters.cs @@ -5,7 +5,7 @@ public class BuildIndexParameters public string StorageAccount { get; set; } public string SourceName { get; set; } public string TargetName { get; set; } - public string TaxonomyFile { get; set; } + public string Taxonomy { get; set; } public string Filter { get; set; } } } diff --git a/DatabaseManager.Services.IndexSqlite/Services/IndexAccess.cs b/DatabaseManager.Services.IndexSqlite/Services/IndexAccess.cs index 9e4d156..490b93e 100644 --- a/DatabaseManager.Services.IndexSqlite/Services/IndexAccess.cs +++ b/DatabaseManager.Services.IndexSqlite/Services/IndexAccess.cs @@ -511,7 +511,7 @@ private IndexFileData GetIndexData(JToken token) private async Task Initialize(ConnectParameters target, ConnectParameters source, BuildIndexParameters idxParms) { - _taxonomy = await _fs.ReadFile("taxonomy", idxParms.TaxonomyFile); + _taxonomy = await _fs.ReadFile("taxonomy", idxParms.Taxonomy); if (source.SourceType == "DataBase") { _sourceAccess = new DBDataAccess(); diff --git a/DatabaseManager.Services.IndexSqlite/mydatabase.db b/DatabaseManager.Services.IndexSqlite/mydatabase.db index 924ad7664a9f08a11e28238ad2798d83ccbae110..3ed48107ad49f91b3b5d8804eaf8fd1990feb384 100644 GIT binary patch delta 9175 zcmeI2dvH@#9>>!cebNV|4+=<(Kp(W>K9byfTcJ(TKugo)kTebWpod+nR*JmWiV8?L zN9!t3MY8NJ1(8)AzIIutj<$q~IOC`b>L@b!2Q#AM$o|3X$T;H~_IK}1Lvu}O|8Vvn zo6Ps*p5MJ`GN1f@=lp)Z+bvsOOW6Y2S&}`sP^ZgyQ>RN`tkb2m^^{o>=N3MW2~Hq5 zk>DhPlL<~Ccmlzx1g8<4PH+annFMDMJdxmRf^!JYB{+}ZNd)H;JelAEf(r>QBDk2~ zDFl}gJeA;S1WzaU7J_FGtS9(Zf(-=UM(|96XAwM`;8KFi2rehMg5Ws>R}x%Bu#w=o z1TzG)1ak!Q1e*vp6Ko+^AXp?=BDk918iH>pcpky?39co$j$kXnHiGR0*Awg@cmcr+ z32q>`k>EuHHxcY4xS3!V!ES;P!HWs@5bQmUTb9-4id7un!ZFnhTg?halN9{8wnd+( zSXrssY^vrgM$ydlU9W9PIa`~Ub4>>jU(NX`=UURbgwwjxq^s&P=Xm59C`rxMC6rHx z>$bI}raqpW$eJ037g)1}4X*B5Q9M^YX5e9{G&mhrucOUnZFU%{4NcBQmjjG`htoM^ z6NBg8>6jes@xEA)W#C04$4D$=VhzDdj@E+NhAPQyWLc4A%sjl&jT|0pi`xT#tKRBa zs&_X!^>(*0m}lBkP-B4i$sbwoKrIbz_0|?&GhC-{aU(Za7c{!8&bF3%Fj+Vg7`(_= zFJC2pvEZpf-{b^C@aj7q#T}2;*-PHN;@x;Iuce>4G*XpP}O<%A(zBq;`$*>nS!*)&#MtQHr`FsvY|xJ(gT631D@SX@?n zvqQhY?Q_)ww%Ph-tGxk!ix_x3J0^I%{6oZpw`R|QqTtRiA4*XOUb=X_@kb!|LdTk8 z2ULOwj;-l<0SNxsc>UrfAgG-y1hvL+K%X*1aNCMlf}BE7aXvE_kD$ZTVD+@X_Z{wW zoE9M?@Vq3c20icDKy=1t@8TEdn9=V-Ojjn|aZjUSKcDxLGRJRjM(g@_DZuivm znV3ePIaNh24E$0<<4KojLs}8JcFULem(-f$7OnXzdYbEF+jXlb}Rtp=D?m z51Y5eJ zO;wh*hZSf(SCh-_clp~pJ8i4hwm)L<`5R%mh?otPRk7<^i^ICm>Zy0QH0a#^Dz-`z zRCu{RAIS*r^1e7pS?+k&BJs?y<<11lv4v&96yto0AZQmw*1HHsNs=9ORJUU}GhBAl zfNchhWjz&b?x`XFi4V3WFYZ+__AXA|`oRf*XsWxdqNmJWr=2SpwZ?GY())dTQ!`)_ zZii(v#h-dY?@P{2&dWKf>sf6G<}$vzz6TheHOHHjpQKAqnj{Z-@{lVJIr5M#4-@4f zOCB=iAwwS0W5I$ZVT( z-<_bl)U&#j!GZZ7gFdtse}+JiIVg__!CZJ-W9s0*?HfT|w&KrcfNqVPP8%Gkae^Xx2#$-^Ut$5%p=v>5}J2)`698_v6KD8Ir8?omN z4wTh^B)IkzXfR@*G&oSPeekw{il=SG?|cvXcjQFAa$=ah@2t~j%CAk;mFm(SPFp(R zL`tEW7DV1~<7GT87*7jES!Kr4g1Fh}cv>)?7C-^?n@9_eeo>H~P@v1cE9;+`ndvK2 z&rPUEUY~d}A(+P){yQyDnj1}8)%@|I`-}u-ry%D^!}9$mhu`Xg%vCKiqscxa*FU3XUA)ANmUYSNVKM*a zF6Lm!|K8VEzB_P6&AQGU`0mQr@A*RqnO$%4#r8VwT*h$GtOx}X6QNkCQWPaoV1-zUP%M3p zMn*I$NegQkf#guxRCGbtb4>tM1zzBkcE+%9u6I~ndPl3(<&B0iKc+g= z%#Tz+fdfw7ywK{cLbI)a&t9@mE}}yI(|wnUzUox*b$(TJsqeHuB$rV8pnliR6?|G_ zI4~tEitp}Nd`hF^MnxLj)U_gwkHC!E1X<*>H#nMavMMU2Joq@nMEXcW6j_sC(S&o0 z&)%e^D97ih-e+M)ZMyAGc=lxo<4<ad6nRFoiFm9|)Sz#0 zIGa_st2RiMG4T>uxrq|5Y#&-O|mk~LQca5-F+zzeXvBfIO zadJ$LfNk<-ufIMTtkDOyREe@hhNLWj%2y<$2IqmAL4N`L6|@)B3)%&rXqP0E}Wn3^AT)7b4rP`4MOQm4dCwdF(@ zd~Nvsh+%iYl%1FrO(rR_v#2+~7?EdLxF;6tqm52yqqSK{ts@n*B#fyxv2Y`0)G={g zw0Gye83mA5FYf=y*R6VJx9_9=#gJAP%-FYcFQnDlx#FQ(BkaeMMgnG0(rOi$CEk7< z2Nhloa~ zK{%1;)MYW^a)M|UIPJ2ibvb!2QTFiB(?8QlNpuw9oif7n`u@v@!RJz2&AZ=LeeUhu z=GIj3xx<(H_2$~&2IHa-EKT`NBM7w(IZ#RXMVlNed+9xH~t6j Cf!{y? delta 4796 zcmeHLdvKK172k#IBiT(hyPF5$5f)Mug~Wa4fh`FnyabYKLV^*58#Tg0!Uj!4m~f65JxVRq$lNQv|mOo+`LqaEIVd!P5j!7d%7oOu?rJo+bEH!KVp6 zUGN!#XA3@4aF^h-1fMN6{070@f_nt_ z3hoo!FL*%ke8CF@FBH5;@CAb3DEKc0|CQhi1uqu7MDRs|FBZI1@G`-d2)s#P+FYp)S6}stk@fw%? zr&Vv!0$+Y!K|%iLzl+xx#`02Zr*v@QknLZ#Z*3QCU)Vmg{l)f??LFJuwm;h5u)St` zNg;SR3q*;_Dye^3_Z%v*cc1koT6OyLbSx`B^QzFM9SZDOZcf(e=gh#GtOc6n3FNv9 ze7Rn)rzk+PzKl8Pt?Av#v}AEjDs?yqXIq!n-Q8G!x69|v^%hax_pwOg>bf2ETU`~w z2KuzDCcDQv7@KPXQ@a!VR7Da>%OUH5+0|=T%w8~Cl~9oD_4xDM)KD|{kL$9GSCNO@6y%19h-3T2*5j$g~0Dd zJc!tEBnNmTf)FH~jF^C*+XeLeR-jY20liQGG`a}rML$keUV|E`p8~Wt=>=DHLTZ98 zRi`7jyLthYxU1)mZgE#HvQ;S&U4zJCSgq41V_2#U#oAD)4IZ5yr*+qBLyk5~*M@X; zm@NF0b%~%cj{ZYs;&L?0HGLnL6_>bFFcoal4^9 zp=k09eWm_JEb4SESxFOO^Zyk5U!~yY>cMwiDZ1BnDdWk%vmUb?NNP%~H2Y0c466@I@#m;C!H->Qzis0t+ZV08@p=;&pNU)ui7lrHnVG@XH#cnU^87Ubftx) z(Zz=boiseHJA?Xn5B^P`KKka2?kv4OJ?+XGbY7~#wbX;3wBG&-%@rgx-HTwSqBHSb!o-8*>JZkx#H$9~$A-F?oUGI6VlX~_iSknB#$>Lt5NvQ3h0mMkdQ7Rk0s)*xAm`NOrGe_er)> zvil|5CE0Gt9+2!o$$l-_Ly|R17M3g`S&L+QBzssgA(@nHuVniqdqlGRlC?^9K(d39 zwMo`4S%+kul07O}*9+`1(}=$Il)3vZ!-AyaXmLYIcT{>K=EHyyz)8S!fad|H051SW z0m=(Qr2>188M*;I00KM?I0QHhcmmK1hytDj^a1(-M*v3wPXT@dcp5MOcm{9`@a*U@ zL+q9qG9qcn<;~0Y%~x=tioFUBYQRSs^yvhFQXa6nxta8vsi>oV!g2aQ%@2L_BZ-W;@2{Gp(cR-bFM+!V-Hi2W+D zFQ6V*)zFyxRB?Vy0d@YPP*sFSqt|wQ^jdu|lfH^WP{&HJSJB8)%pNdDt;F|X*0c#I zel%vJh0OyFif>RJTY?rUj$k(YY|u%r=AeT*+JO3_1CE|SOtOzbWxg2{7uI1m-ix)8 zodZ@%y%}q)4`{3t>`G$Tb#XlMmY`u$ppfkMz>}jxVMm%r)!wVx=2NlXM;8ZU8N^OE zn)H5J^h`L17WRj&wEqFH{`}947Rs|lP5ON5Y~`8Mu`Ot)rl&$Cy6|KSqHl$mKb%** zst9x`IzcfbjWol_(8)n>wGCJ(^}|LJeU;N-p^iw%NGFn_CSpI)>D?6H6Uvz6Eug;k zaK=eF%7} zKvujyXhft#C@Sik*8nx5SRF_H^t6KT%o7NZS|cM$jj(5f27Q1!JGM-rclJU{+v|w7 z?{uSs+J=<<;jpp$hNzJ$+P6R+t%K!+^|*lg4asn0+2@T;+PJ>eKu!C$nCKG|M2_u* zq|fcx9lH&)_;R4sA?P|rF=Nl;2p+4#Qn4AdiqSOnJeWLm3~MC^gHvegjwg`g$MM)+ z(uzV^35gviATj%9cz9k^3C95?f1kmue=X?xp8;ZBAlc?X@)yJ)b*eqZcSR%cdB}bwID=Apq28PX#N(xe1}hcb6|{P^`d530R#|SX0ckBs+g0aU z#o^68G7!$x7g4+~lu0kQ4_I7Rb{@YbT*GSO``vW%w{4lU;T}XUTnkNKU4cY@rE$PA z$zMn(dyoMay0L8gpb_SEz*9e*?}b-dkmf1JV@AD?W=Rlz{22oCl`U!^|N2%Zl{A9# z?HZ+8zXo|p6}H)77I+cY-W3BcfZP%~1 zeSZCTZ3o%iN(x-;^hm1WZVum!ose-jDaH7Cn_wKvPUT;Zs+iq>w)63e`s{qyJ1@1d_}D8o)Y*X@Y8+@)=c- z5h=04e&t(N{k?kWJAy@u@#d_3NCdH`{EczhO8kwHX1-+eqQy07>R%b=^5Ffh98a$H aJ)gKbbUVK0mA^6aLecj|jbpLzhW`T|^~DeX