From f1cf1ae1027c64af7047404e2035371c4ebc78b2 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Thu, 6 Oct 2022 15:34:13 +0700 Subject: [PATCH 1/6] Added debug capability to LfMerge This is possible by supplying pbuild with the "development" arg, which signals to docker to download vsdbg to the container. Also adding the config for VS Code to attach to a process in this container. There are still some kinks to work out. Breakpoints are set in source normally, but during debug it's pulling source files from Nuget. So local changes aren't displayed when stepping through. Also, it would be nice to get symbol caching up and working, as loading everything takes longer than actually running LfMerge. A fallback is to load only our own DLL's. --- .vscode/launch.json | 35 +++++++++++++++++++++++++++++++++-- Dockerfile.finalresult | 13 ++++++++++++- pbuild.sh | 3 ++- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1f94266c..d42fc710 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -60,7 +60,38 @@ { "name": ".NET Core Attach", "type": "coreclr", + "request": "attach" + }, + { + "name": "Docker Attach", + "type": "coreclr", "request": "attach", - }, - ] + "processId": "${command:pickRemoteProcess}", + "justMyCode": false, + "requireExactSource": false, + // "symbolOptions": { + // "searchPaths": [ ], + // "searchMicrosoftSymbolServer": false, + // "searchNuGetOrgSymbolServer": false, + // "moduleFilter": { + // "mode": "loadOnlyIncluded", + // "includedModules": [ + // "LfMerge*.dll", + // "SIL*.dll", + // ] + // } + // }, + "pipeTransport": { + "pipeProgram": "docker", + "pipeArgs": [ + "exec", + "-i", + "lfmerge" + ], + "debuggerPath": "/vsdbg/vsdbg", + "pipeCwd": "${workspaceRoot}", + "quoteArgs": false + }, + } + ] } \ No newline at end of file diff --git a/Dockerfile.finalresult b/Dockerfile.finalresult index f5a52356..cbe44251 100644 --- a/Dockerfile.finalresult +++ b/Dockerfile.finalresult @@ -1,7 +1,8 @@ # syntax=docker/dockerfile:experimental ARG DbVersion=7000072 +ARG Environment=production -FROM ghcr.io/sillsdev/lfmerge-base:runtime +FROM ghcr.io/sillsdev/lfmerge-base:runtime AS lfmerge-base-runtime # install LFMerge prerequisites # tini - PID 1 handler @@ -13,6 +14,16 @@ RUN apt-get update \ && apt-get install --yes --no-install-recommends tini python iputils-ping inotify-tools less vim-tiny \ && rm -rf /var/lib/apt/lists/* +FROM lfmerge-base-runtime AS lfmerge-base-runtime-development + +RUN apt update && \ + apt install unzip && \ + apt install curl -y && \ + curl -sSL https://aka.ms/getvsdbgsh | /bin/sh /dev/stdin -v latest -l /vsdbg + +FROM lfmerge-base-runtime AS lfmerge-base-runtime-production +FROM lfmerge-base-runtime-${Environment} + ADD tarball/lfmerge* / RUN mkdir -m 02775 -p /var/lib/languageforge/lexicon/sendreceive/syncqueue /var/lib/languageforge/lexicon/sendreceive/webwork /var/lib/languageforge/lexicon/sendreceive/Templates /var/lib/languageforge/lexicon/sendreceive/state && chown -R www-data:www-data /var/lib/languageforge diff --git a/pbuild.sh b/pbuild.sh index ccaa41cd..4912b8af 100755 --- a/pbuild.sh +++ b/pbuild.sh @@ -5,6 +5,7 @@ set -e # These are arrays; see https://www.gnu.org/software/bash/manual/html_node/Arrays.html DBMODEL_VERSIONS=(7000072) HISTORICAL_VERSIONS=(7000068 7000069 7000070) +Environment=${1:-production} # In the future when we have more than one model version, we may want to use GNU parallel for building. # ATTENTION: If GNU parallel is desired, uncomment the below (until the "ATTENTION: Stop uncommenting here" line): @@ -80,4 +81,4 @@ for DbVersion in ${DBMODEL_VERSIONS[@]}; do lfmerge-build-${DbVersion} done -time docker build -t ghcr.io/sillsdev/lfmerge -f Dockerfile.finalresult . +time docker build -t ghcr.io/sillsdev/lfmerge -f Dockerfile.finalresult . --build-arg Environment=${Environment} From 3a464f63b20aeedaaac8a98ad6a03c6a46a48c7b Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Wed, 2 Nov 2022 11:18:47 -0500 Subject: [PATCH 2/6] Updated readme for debugging capabilities. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8bff0777..918415f6 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,9 @@ For each DbVersion that LfMerge supports, we build a different lfmerge binary. W ## Debugging -Debugging is possible, in some form, with the C# extension in VS Code. Run pbuild.sh (which creates the environment used by the debugger), set your breakpoints, and run the .NET Core Launch task. Due to the complex nature of the software, which necessitates the use of pbuild.sh, for example, there may be custom setup required to progress far enough to reach your breakpoints, depending on where they are. Debugging will launch and use LfQueueManager as its entry point. +Debugging is possible through the "Docker Attach" launch configuration in VS Code. All the expected debugging features are there, including breakpoints, call stack, and source viewing. There is some work left to be done, however: the source used is pulled from GitHub, excluding local changes. The breakpoints work, simply swap between the file pulled from GitHub and the local, comparing line numbers. + +If you get your project put on HOLD and need to recover quickly, use the "Attach to Running Container..." command to attach to the "lfmerge" container and change the project state manually (to IDLE) located in `/var/lib/languageforge/lexicon/sendreceive/state/.state`. ## Testing locally From cdc42a20bca21caaa1ef2af6fd1dc3ca2b446445 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Wed, 9 Nov 2022 13:26:16 -0600 Subject: [PATCH 3/6] Using GUID as key for LfOptionListItem. Also using GUID's for LfStringField and LfStringArrayField, since there a few places in the Convert* classes that find the LCM item from the LfString* value. We need to shift our class design to be slightly more controlled. This will reduce bugs and make the code easier to read. I made some constructors private so that we only use static create methods, and we require a GUID to be passed in so that we can better depend on the object being fully constructed after calling its construction method (read: partially constructed objects are bad). In LfStringField, I replaced the Guids property with LcmGuid. The former didn't appear to be used, so I was cleaning up. If we need to revert that part, we can, but it's not great API. To uplift existing data to GUID's, I changed the setter in LfOptionListItem to only allow GUID's and to ignore incoming values that aren't GUID's. If a GUID is not provided, it will use the existing Guid property. I opted for this over a separate migration action, since that would require manually setting each property and subproperty for each entry. This action would be a big effort and quite bugprone, where changing the setter enforces the change at the one place all these properties use. More properties were affected than just the option list items, since there are some cases where LF needs to back-convert from a simple string field to its LCM item. Therefore, many of the MultiText properties that are set during entry conversion use the entry GUID, not any GUID for that specific field. I'm not crazy about this, so maybe a further revision will be needed. The full feature is not yet working, but I haven't worked on the Language Forge side yet. --- .../Actions/SynchronizeActionTests.cs | 14 +-- src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs | 63 ++++++------- .../Lcm/TransferLcmToMongoActionTests.cs | 3 - .../Lcm/TransferMongoToLcmActionTests.cs | 28 +++--- .../ConvertLcmToMongoCustomField.cs | 18 ++-- .../ConvertLcmToMongoLexicon.cs | 90 +++++++++---------- .../ConvertLcmToMongoOptionList.cs | 90 ++----------------- .../ConvertMongoToLcmOptionList.cs | 51 ++++------- .../LanguageForge/Model/LfFieldBase.cs | 2 +- .../LanguageForge/Model/LfMultiText.cs | 16 ++-- .../LanguageForge/Model/LfOptionListItem.cs | 26 +++++- .../LanguageForge/Model/LfStringArrayField.cs | 22 ++--- .../LanguageForge/Model/LfStringField.cs | 20 ++--- 13 files changed, 173 insertions(+), 270 deletions(-) diff --git a/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs b/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs index 1cd04787..88eb1c52 100644 --- a/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs +++ b/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs @@ -164,8 +164,8 @@ public void SynchronizeAction_LFDataChanged_GlossChanged() IEnumerable originalMongoData = _mongoConnection.GetLfLexEntries(); LfLexEntry lfEntry = originalMongoData.First(e => e.Guid == _testEntryGuid); string unchangedGloss = lfEntry.Senses[0].Gloss["en"].Value; - string lfChangedGloss = unchangedGloss + " - changed in LF"; - lfEntry.Senses[0].Gloss["en"].Value = lfChangedGloss; + var lfChangedGloss = LfStringField.CreateFrom(unchangedGloss + " - changed in LF", new Guid()); + lfEntry.Senses[0].Gloss["en"] = lfChangedGloss; _mongoConnection.UpdateRecord(_lfProject, lfEntry); _lDProject = new LanguageDepotMock(testProjectCode, _lDSettings); @@ -186,7 +186,7 @@ public void SynchronizeAction_LFDataChanged_GlossChanged() Assert.That(GetGlossFromLanguageDepot(_testEntryGuid, 2), Is.EqualTo(lfChangedGloss)); } - [Test, Explicit("Superceeded by later tests")] + [Test, Explicit("Superceded by later tests")] public void SynchronizeAction_LDDataChanged_GlossChanged() { // Setup @@ -246,8 +246,8 @@ public void SynchronizeAction_LFDataChangedLDDataChanged_LFWins() string unchangedGloss = lfEntry.Senses[0].Gloss["en"].Value; string fwChangedGloss = unchangedGloss + " - changed in FW"; - string lfChangedGloss = unchangedGloss + " - changed in LF"; - lfEntry.Senses[0].Gloss["en"].Value = lfChangedGloss; + var lfChangedGloss = LfStringField.CreateFrom(unchangedGloss + " - changed in LF", new Guid()); + lfEntry.Senses[0].Gloss["en"] = lfChangedGloss; lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _mongoConnection.UpdateRecord(_lfProject, lfEntry); @@ -436,7 +436,7 @@ public void SynchronizeAction_LFDataChangedLDEntryDeleted_LFWins() const string lfCreatedGloss = "new English gloss - added in LF"; const string fwChangedGloss = "English gloss - changed in FW"; // LF adds a gloss to the entry that LD is deleting - lfEntry.Senses[0].Gloss = LfMultiText.FromSingleStringMapping("en", lfCreatedGloss); + lfEntry.Senses[0].Gloss = LfMultiText.FromSingleStringMapping("en", lfCreatedGloss, new Guid()); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _mongoConnection.UpdateRecord(_lfProject, lfEntry); @@ -480,7 +480,7 @@ public void SynchronizeAction_LFDataChangedLDOtherDataChanged_ModifiedDateUpdate var originalLfDateModified = lfEntry.DateModified; var originalLfAuthorInfoModifiedDate = lfEntry.AuthorInfo.ModifiedDate; - lfEntry.Note = LfMultiText.FromSingleStringMapping("en", "A note from LF"); + lfEntry.Note = LfMultiText.FromSingleStringMapping("en", "A note from LF", new Guid()); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _mongoConnection.UpdateRecord(_lfProject, lfEntry); diff --git a/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs b/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs index 7902a85c..6633c619 100644 --- a/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs @@ -181,7 +181,8 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInEntries() string vernacularWS = cache.ServiceLocator.WritingSystemManager.GetStrFromWs(cache.DefaultVernWs); string originalLexeme = originalLfEntry.Lexeme[vernacularWS].Value; string changedLexeme = "Changed lexeme for this test"; - originalLfEntry.Lexeme[vernacularWS].Value = changedLexeme; + var originalValue = LfStringField.CreateFrom(changedLexeme, new Guid()); + originalLfEntry.Lexeme[vernacularWS] = originalValue; originalLfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalLfEntry); @@ -192,7 +193,7 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInEntries() // Exercise SutMongoToLcm.Run(lfProject); string changedLexemeDuringUpdate = "This value should be overwritten by LcmToMongo"; - originalLfEntry.Lexeme[vernacularWS].Value = changedLexemeDuringUpdate; + originalLfEntry.Lexeme[vernacularWS] = LfStringField.CreateFrom(changedLexemeDuringUpdate, new Guid()); originalLfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalLfEntry); SutLcmToMongo.Run(lfProject); @@ -220,7 +221,7 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInEntries() Assert.That(lfEntry.Lexeme[vernacularWS].Value, Is.Not.EqualTo(changedLexemeDuringUpdate)); Assert.That(lfEntry.Lexeme[vernacularWS].Value, Is.EqualTo(changedLexeme)); - originalLfEntry.Lexeme[vernacularWS].Value = originalLexeme; + originalLfEntry.Lexeme[vernacularWS] = originalValue; differencesByName = GetMongoDifferences(originalLfEntry.ToBsonDocument(), lfEntry.ToBsonDocument()); differencesByName.Remove("lexeme"); differencesByName.Remove("dateModified"); @@ -266,20 +267,20 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInSenses() LfLexEntry originalEntry = originalData.FirstOrDefault(e => e.Guid.ToString() == TestEntryGuidStr); Assert.That(originalEntry.Senses.Count, Is.EqualTo(2)); - string originalSense0Definition = originalEntry.Senses[0].Definition["en"].Value; - string originalSense1Definition = originalEntry.Senses[1].Definition["en"].Value; - string changedSense0Definition = "Changed sense0 definition for this test"; - string changedSense1Definition = "Changed sense1 definition for this test"; - originalEntry.Senses[0].Definition["en"].Value = changedSense0Definition; - originalEntry.Senses[1].Definition["en"].Value = changedSense1Definition; + var originalSense0Definition = originalEntry.Senses[0].Definition["en"]; + var originalSense1Definition = originalEntry.Senses[1].Definition["en"]; + var changedSense0Definition = LfStringField.CreateFrom("Changed sense0 definition for this test", new Guid()); + var changedSense1Definition = LfStringField.CreateFrom("Changed sense1 definition for this test", new Guid()); + originalEntry.Senses[0].Definition["en"] = changedSense0Definition; + originalEntry.Senses[1].Definition["en"] = changedSense1Definition; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalEntry); // Exercise SutMongoToLcm.Run(lfProject); - string changedDefinitionDuringUpdate = "This value should be overwritten by LcmToMongo"; - originalEntry.Senses[0].Definition["en"].Value = changedDefinitionDuringUpdate; - originalEntry.Senses[1].Definition["en"].Value = changedDefinitionDuringUpdate; + var changedDefinitionDuringUpdate = LfStringField.CreateFrom("This value should be overwritten by LcmToMongo", new Guid()); + originalEntry.Senses[0].Definition["en"] = changedDefinitionDuringUpdate; + originalEntry.Senses[1].Definition["en"] = changedDefinitionDuringUpdate; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalEntry); @@ -318,8 +319,8 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInSenses() Assert.That(lfEntry.Senses[0].Definition["en"].Value, Is.EqualTo(changedSense0Definition)); Assert.That(lfEntry.Senses[1].Definition["en"].Value, Is.EqualTo(changedSense1Definition)); - originalEntry.Senses[0].Definition["en"].Value = originalSense0Definition; - originalEntry.Senses[1].Definition["en"].Value = originalSense1Definition; + originalEntry.Senses[0].Definition["en"] = originalSense0Definition; + originalEntry.Senses[1].Definition["en"] = originalSense1Definition; IDictionary> differencesByName = GetMongoDifferences(originalEntry.Senses[0].ToBsonDocument(), lfEntry.Senses[0].ToBsonDocument()); differencesByName.Remove("definition"); @@ -378,20 +379,20 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInExample() Assert.That(originalEntry.Senses.Count, Is.EqualTo(2)); Assert.That(originalEntry.Senses[0].Examples.Count, Is.EqualTo(2)); - string originalSense0Example0Translation = originalEntry.Senses[0].Examples[0].Translation["en"].Value; - string originalSense0Example1Translation = originalEntry.Senses[0].Examples[1].Translation["en"].Value; - string changedSense0Example0Translation = "Changed sense0 example0 sentence for this test"; - string changedSense0Example1Translation = "Changed sense0 example1 sentence for this test"; - originalEntry.Senses[0].Examples[0].Translation["en"].Value = changedSense0Example0Translation; - originalEntry.Senses[0].Examples[1].Translation["en"].Value = changedSense0Example1Translation; + var originalSense0Example0Translation = originalEntry.Senses[0].Examples[0].Translation["en"]; + var originalSense0Example1Translation = originalEntry.Senses[0].Examples[1].Translation["en"]; + var changedSense0Example0Translation = LfStringField.CreateFrom("Changed sense0 example0 sentence for this test", new Guid()); + var changedSense0Example1Translation = LfStringField.CreateFrom("Changed sense0 example1 sentence for this test", new Guid()); + originalEntry.Senses[0].Examples[0].Translation["en"] = changedSense0Example0Translation; + originalEntry.Senses[0].Examples[1].Translation["en"] = changedSense0Example1Translation; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalEntry); // Exercise SutMongoToLcm.Run(lfProject); - string changedTranslationDuringUpdate = "This value should be overwritten by LcmToMongo"; - originalEntry.Senses[0].Examples[0].Translation["en"].Value = changedTranslationDuringUpdate; - originalEntry.Senses[0].Examples[1].Translation["en"].Value = changedTranslationDuringUpdate; + var changedTranslationDuringUpdate = LfStringField.CreateFrom("This value should be overwritten by LcmToMongo", new Guid()); + originalEntry.Senses[0].Examples[0].Translation["en"] = changedTranslationDuringUpdate; + originalEntry.Senses[0].Examples[1].Translation["en"] = changedTranslationDuringUpdate; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalEntry); @@ -436,8 +437,8 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInExample() Assert.That(lfEntry.Senses[0].Examples[0].Translation["en"].Value, Is.EqualTo(changedSense0Example0Translation)); Assert.That(lfEntry.Senses[0].Examples[1].Translation["en"].Value, Is.EqualTo(changedSense0Example1Translation)); - originalEntry.Senses[0].Examples[0].Translation["en"].Value = originalSense0Example0Translation; - originalEntry.Senses[0].Examples[1].Translation["en"].Value = originalSense0Example1Translation; + originalEntry.Senses[0].Examples[0].Translation["en"] = originalSense0Example0Translation; + originalEntry.Senses[0].Examples[1].Translation["en"] = originalSense0Example1Translation; IDictionary> differencesByName = GetMongoDifferences(originalEntry.Senses[0].Examples[0].ToBsonDocument(), lfEntry.Senses[0].Examples[0].ToBsonDocument()); differencesByName.Remove("translation"); @@ -472,7 +473,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewEntry() newEntry.Guid = Guid.NewGuid(); string vernacularWS = lfProject.FieldWorksProject.Cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = new DateTime(); newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -552,8 +553,8 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewSense() Assert.That(lfEntry.Senses.Count, Is.EqualTo(2)); LfSense newSense = new LfSense(); newSense.Guid = Guid.NewGuid(); - newSense.Definition = LfMultiText.FromSingleStringMapping(vernacularWS, newDefinition); - newSense.PartOfSpeech = LfStringField.FromString(newPartOfSpeech); + newSense.Definition = LfMultiText.FromSingleStringMapping(vernacularWS, newDefinition, newSense.Guid.Value); + newSense.PartOfSpeech = LfStringField.CreateFrom(newPartOfSpeech, newSense.Guid.Value); lfEntry.Senses.Add(newSense); Assert.That(lfEntry.Senses.Count, Is.EqualTo(3)); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -642,8 +643,8 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewExample() Assert.That(lfSense.Examples.Count, Is.EqualTo(2)); LfExample newExample = new LfExample(); newExample.Guid = Guid.NewGuid(); - newExample.Sentence = LfMultiText.FromSingleStringMapping(vernacularWS, newSentence); - newExample.Translation = LfMultiText.FromSingleStringMapping(vernacularWS, newTranslation); + newExample.Sentence = LfMultiText.FromSingleStringMapping(vernacularWS, newSentence, newExample.Guid.Value); + newExample.Translation = LfMultiText.FromSingleStringMapping(vernacularWS, newTranslation, newExample.Guid.Value); lfSense.Examples.Add(newExample); Assert.That(lfSense.Examples.Count, Is.EqualTo(3)); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -736,7 +737,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewPicture() Assert.That(lfSense.Pictures.Count, Is.EqualTo(1)); LfPicture newPicture = new LfPicture(); newPicture.Guid = Guid.NewGuid(); - newPicture.Caption = LfMultiText.FromSingleStringMapping(vernacularWS, newCaption); + newPicture.Caption = LfMultiText.FromSingleStringMapping(vernacularWS, newCaption, newPicture.Guid.Value); newPicture.FileName = newFilename; lfSense.Pictures.Add(newPicture); Assert.That(lfSense.Pictures.Count, Is.EqualTo(2)); diff --git a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs index db54a05d..c07f56d8 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferLcmToMongoActionTests.cs @@ -35,7 +35,6 @@ private IEnumerable DefaultGrammarItems(int howMany) string abbrev = PartOfSpeechMasterList.FlatPosAbbrevs[guidStr]; yield return new LfOptionListItem { Guid = Guid.Parse(guidStr), - Key = abbrev, Abbreviation = abbrev, Value = name, }; @@ -242,7 +241,6 @@ public void Action_WithPreviousMongoGrammarWithMatchingGuids_ShouldBeUpdatedFrom Guid g = itemForTest.Guid.Value; itemForTest.Abbreviation = "Different abbreviation"; itemForTest.Value = "Different name"; - itemForTest.Key = "Different key"; _conn.UpdateMockOptionList(lfGrammar); // Exercise @@ -258,7 +256,6 @@ public void Action_WithPreviousMongoGrammarWithMatchingGuids_ShouldBeUpdatedFrom Assert.That(itemForTest, Is.Not.Null); Assert.That(itemForTest.Abbreviation, Is.Not.EqualTo("Different abbreviation")); Assert.That(itemForTest.Value, Is.Not.EqualTo("Different name")); - Assert.That(itemForTest.Key, Is.EqualTo("Different key")); // NOTE: Is.EqualTo, because keys shouldn't be updated } } } diff --git a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs index df306d5c..dde7408d 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs @@ -95,8 +95,8 @@ public void Action_ChangedWithSampleData_ShouldUpdatePictures() Assert.That(entry.SensesOS[0].PicturesOS[1].PictureFileRA.InternalPath.ToString(), Is.EqualTo(expectedExternalFileName)); - LfMultiText expectedNewCaption = ConvertLcmToMongoLexicon. - ToMultiText(entry.SensesOS[0].PicturesOS[0].Caption, cache.ServiceLocator.WritingSystemManager); + LfMultiText expectedNewCaption = LfMultiText.FromLcmMultiString( + entry.SensesOS[0].PicturesOS[0].Caption, entry.Guid, cache.ServiceLocator.WritingSystemFactory); int expectedNumOfNewCaptions = expectedNewCaption.Count(); Assert.That(expectedNumOfNewCaptions, Is.EqualTo(2)); string expectedNewVernacularCaption = expectedNewCaption["qaa-x-kal"].Value; @@ -207,7 +207,7 @@ public void Action_WithOneNewEntry_ShouldCountOneAdded() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -237,7 +237,7 @@ public void Action_WithOneModifiedEntry_ShouldCountOneModified() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -289,7 +289,7 @@ public void Action_WithTwoNewEntries_ShouldCountTwoAdded() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -298,7 +298,7 @@ public void Action_WithTwoNewEntries_ShouldCountTwoAdded() LfLexEntry newEntry2 = new LfLexEntry(); newEntry2.Guid = Guid.NewGuid(); string newLexeme2 = "new lexeme #2 for this test"; - newEntry2.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme2); + newEntry2.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme2, newEntry2.Guid.Value); newEntry2.AuthorInfo = new LfAuthorInfo(); newEntry2.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry2.AuthorInfo.ModifiedDate = newEntry2.AuthorInfo.CreatedDate; @@ -328,7 +328,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -336,7 +336,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified() Guid kenGuid = Guid.Parse(KenEntryGuidStr); LfLexEntry kenEntry = _conn.GetLfLexEntryByGuid(kenGuid); string changedLexeme2 = "modified lexeme #2 for this test"; - kenEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2); + kenEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2, kenGuid); kenEntry.AuthorInfo = new LfAuthorInfo(); kenEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(kenEntry); @@ -392,7 +392,7 @@ public void Action_WithOneNewEntry_ShouldNotCountThatNewEntryOnSecondRun() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -433,7 +433,7 @@ public void Action_WithOneModifiedEntry_ShouldNotCountThatModifiedEntryOnSecondR LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -507,7 +507,7 @@ public void Action_RunTwiceWithOneNewEntryEachTime_ShouldCountTwoAddedInTotal() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -528,7 +528,7 @@ public void Action_RunTwiceWithOneNewEntryEachTime_ShouldCountTwoAddedInTotal() newEntry = new LfLexEntry(); newEntry.Guid = Guid.NewGuid(); newLexeme = "second new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -560,7 +560,7 @@ public void Action_RunTwiceWithTheSameEntryModifiedEachTime_ShouldCountTwoModifi LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -578,7 +578,7 @@ public void Action_RunTwiceWithTheSameEntryModifiedEachTime_ShouldCountTwoModifi // Setup second run string changedLexeme2 = "second modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2, entryGuid); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs index 3a736398..6adcf34e 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs @@ -326,6 +326,8 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp CellarPropertyType LcmFieldType = (CellarPropertyType)LcmMetaData.GetFieldType(flid); var dataGuids = new List(); + var guid = data.get_GuidProp(hvo, flid); + // Valid field types in Lcm are GenDate, Integer, String, OwningAtomic, ReferenceAtomic, and ReferenceCollection, so that's all we implement. switch (LcmFieldType) { @@ -335,7 +337,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp // LF wants single-string fields in the format { "ws": { "value": "contents" } } fieldValue = String.IsNullOrEmpty(genDateStr) ? null : LfMultiText.FromSingleStringMapping( - MagicStrings.LanguageCodeForGenDateFields, genDateStr).AsBsonDocument(); + MagicStrings.LanguageCodeForGenDateFields, genDateStr, guid).AsBsonDocument(); break; // When parsing, will use GenDate.TryParse(str, out genDate) @@ -346,7 +348,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp else // LF wants single-string fields in the format { "ws": { "value": "contents" } } fieldValue = LfMultiText.FromSingleStringMapping( - MagicStrings.LanguageCodeForIntFields, fieldValue.AsInt32.ToString()).AsBsonDocument(); + MagicStrings.LanguageCodeForIntFields, fieldValue.AsInt32.ToString(), guid).AsBsonDocument(); break; case CellarPropertyType.OwningAtomic: @@ -363,7 +365,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp case CellarPropertyType.MultiUnicode: ITsMultiString tss = data.get_MultiStringProp(hvo, flid); if (tss != null && tss.StringCount > 0) - fieldValue = LfMultiText.FromMultiITsString(tss, servLoc.WritingSystemManager).AsBsonDocument(); + fieldValue = LfMultiText.FromMultiITsString(tss, guid, servLoc.WritingSystemManager).AsBsonDocument(); break; case CellarPropertyType.OwningCollection: case CellarPropertyType.OwningSequence: @@ -376,7 +378,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp else { fieldValue = new BsonDocument("values", innerValues); - fieldGuid = new BsonArray(dataGuids.Select(guid => guid.ToString())); + fieldGuid = new BsonArray(dataGuids.Select(g => g.ToString())); } break; @@ -385,7 +387,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp if (iTsValue == null || String.IsNullOrEmpty(iTsValue.Text)) fieldValue = null; else - fieldValue = LfMultiText.FromSingleITsString(iTsValue, servLoc.WritingSystemManager).AsBsonDocument(); + fieldValue = LfMultiText.FromSingleITsString(iTsValue, guid, servLoc.WritingSystemManager).AsBsonDocument(); break; default: fieldValue = null; @@ -489,9 +491,9 @@ private BsonValue GetCustomReferencedObject(int hvo, int flid, servLoc.WritingSystemManager, LcmMetaData, cache.DefaultUserWs); else if (referencedObject is ICmPossibility) { - //return GetCustomListValues((ICmPossibility)referencedObject, flid); - string listCode = GetParentListCode(flid); - return new BsonString(listConverters[listCode].LfItemKeyString((ICmPossibility)referencedObject, _wsEn)); + ICmPossibility poss = (ICmPossibility) referencedObject; + var abbreviation = ConvertLcmToMongoTsStrings.TextFromTsString(poss.Abbreviation.get_String(_wsEn), servLoc.WritingSystemFactory); + return new BsonString(abbreviation); } else return null; diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index 33792518..a43010f6 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -187,32 +187,28 @@ private T GetInstance() where T : class return ServiceLocator.GetInstance(); } - private LfMultiText ToMultiText(IMultiAccessorBase LcmMultiString) + private LfMultiText ToMultiText(IMultiAccessorBase LcmMultiString, Guid lcmGuid) { if (LcmMultiString == null) return null; - return LfMultiText.FromLcmMultiString(LcmMultiString, ServiceLocator.WritingSystemManager); - } - - public static LfMultiText ToMultiText(IMultiAccessorBase LcmMultiString, ILgWritingSystemFactory LcmWritingSystemManager) - { - if ((LcmMultiString == null) || (LcmWritingSystemManager == null)) return null; - return LfMultiText.FromLcmMultiString(LcmMultiString, LcmWritingSystemManager); + return LfMultiText.FromLcmMultiString(LcmMultiString, lcmGuid, ServiceLocator.WritingSystemManager); } private LfStringField ToStringField(string listCode, ICmPossibility LcmPoss) { - return LfStringField.FromString(ListConverters[listCode].LfItemKeyString(LcmPoss, _wsEn)); + var abbreviation = ConvertLcmToMongoTsStrings.TextFromTsString(LcmPoss.Abbreviation.get_String(_wsEn), ServiceLocator.WritingSystemFactory); + return LfStringField.CreateFrom(abbreviation, LcmPoss.Guid); } private LfStringArrayField ToStringArrayField(string listCode, IEnumerable LcmPossCollection) { - return LfStringArrayField.FromStrings(ListConverters[listCode].LfItemKeyStrings(LcmPossCollection, _wsEn)); + var strings = LcmPossCollection.Select(p => ToStringField(listCode, p)); + return LfStringArrayField.CreateFrom(strings); } // Special case: LF sense Status field is a StringArray, but Lcm sense status is single possibility private LfStringArrayField ToStringArrayField(string listCode, ICmPossibility LcmPoss) { - return LfStringArrayField.FromSingleString(ListConverters[listCode].LfItemKeyString(LcmPoss, _wsEn)); + return LfStringArrayField.CreateFrom(new List() { ToStringField(listCode, LcmPoss) }); } /// @@ -233,7 +229,7 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) if (LcmLexeme == null) lfEntry.Lexeme = null; else - lfEntry.Lexeme = ToMultiText(LcmLexeme.Form); + lfEntry.Lexeme = ToMultiText(LcmLexeme.Form, LcmLexeme.Guid); // Other fields of LcmLexeme (AllomorphEnvironments, LiftResidue, MorphTypeRA, etc.) not mapped // Fields below in alphabetical order by ILexSense property, except for Lexeme @@ -241,11 +237,11 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) { // Do nothing; LanguageForge doesn't currently handle allomorphs, so we don't convert them } - lfEntry.EntryBibliography = ToMultiText(LcmEntry.Bibliography); + lfEntry.EntryBibliography = ToMultiText(LcmEntry.Bibliography, LcmEntry.Guid); // TODO: Consider whether to use LcmEntry.CitationFormWithAffixType instead // (which would produce "-s" instead of "s" for the English plural suffix, for instance) - lfEntry.CitationForm = ToMultiText(LcmEntry.CitationForm); - lfEntry.Note = ToMultiText(LcmEntry.Comment); + lfEntry.CitationForm = ToMultiText(LcmEntry.CitationForm, LcmEntry.Guid); + lfEntry.Note = ToMultiText(LcmEntry.Comment, LcmEntry.Guid); // DateModified and DateCreated can be confusing, because LF and Lcm are doing two different // things with them. In Lcm, there is just one DateModified and one DateCreated; simple. But @@ -284,19 +280,15 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) #endif if (LcmEtymology != null) { - lfEntry.Etymology = ToMultiText(LcmEtymology.Form); - lfEntry.EtymologyComment = ToMultiText(LcmEtymology.Comment); - lfEntry.EtymologyGloss = ToMultiText(LcmEtymology.Gloss); -#if DBVERSION_7000068 - lfEntry.EtymologySource = LfMultiText.FromSingleStringMapping(AnalysisWritingSystem.Id, LcmEtymology.Source); -#else - lfEntry.EtymologySource = ToMultiText(LcmEtymology.LanguageNotes); -#endif + lfEntry.Etymology = ToMultiText(LcmEtymology.Form, LcmEtymology.Guid); + lfEntry.EtymologyComment = ToMultiText(LcmEtymology.Comment, LcmEtymology.Guid); + lfEntry.EtymologyGloss = ToMultiText(LcmEtymology.Gloss, LcmEtymology.Guid); + lfEntry.EtymologySource = ToMultiText(LcmEtymology.LanguageNotes, LcmEtymology.Guid); // LcmEtymology.LiftResidue not mapped } lfEntry.Guid = LcmEntry.Guid; // LcmEntry.LIFTid not mapped (changed 2019-10 by RM since the LiftId in LF is not useful: see LF-378) - lfEntry.LiteralMeaning = ToMultiText(LcmEntry.LiteralMeaning); + lfEntry.LiteralMeaning = ToMultiText(LcmEntry.LiteralMeaning, LcmEntry.Guid); if (LcmEntry.PrimaryMorphType != null) { lfEntry.MorphologyType = LcmEntry.PrimaryMorphType.NameHierarchyString; } @@ -305,17 +297,17 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) if (LcmEntry.PronunciationsOS.Count > 0) { ILexPronunciation LcmPronunciation = LcmEntry.PronunciationsOS.First(); - lfEntry.Pronunciation = ToMultiText(LcmPronunciation.Form); - lfEntry.CvPattern = LfMultiText.FromSingleITsString(LcmPronunciation.CVPattern, ServiceLocator.WritingSystemFactory); - lfEntry.Tone = LfMultiText.FromSingleITsString(LcmPronunciation.Tone, ServiceLocator.WritingSystemFactory); + lfEntry.Pronunciation = ToMultiText(LcmPronunciation.Form, LcmPronunciation.Guid); + lfEntry.CvPattern = LfMultiText.FromSingleITsString(LcmPronunciation.CVPattern, LcmPronunciation.Guid, ServiceLocator.WritingSystemFactory); + lfEntry.Tone = LfMultiText.FromSingleITsString(LcmPronunciation.Tone, LcmPronunciation.Guid, ServiceLocator.WritingSystemFactory); // TODO: Map LcmPronunciation.MediaFilesOS properly (converting video to sound files if necessary) lfEntry.Location = ToStringField(LocationListCode, LcmPronunciation.LocationRA); } - lfEntry.EntryRestrictions = ToMultiText(LcmEntry.Restrictions); + lfEntry.EntryRestrictions = ToMultiText(LcmEntry.Restrictions, LcmEntry.Guid); if (lfEntry.Senses == null) // Shouldn't happen, but let's be careful lfEntry.Senses = new List(); lfEntry.Senses.AddRange(LcmEntry.SensesOS.Select(LcmSenseToLfSense)); - lfEntry.SummaryDefinition = ToMultiText(LcmEntry.SummaryDefinition); + lfEntry.SummaryDefinition = ToMultiText(LcmEntry.SummaryDefinition, LcmEntry.Guid); BsonDocument customFieldsAndGuids = _convertCustomField.GetCustomFieldsForThisCmObject(LcmEntry, "entry", ListConverters); BsonDocument customFieldsBson = customFieldsAndGuids["customFields"].AsBsonDocument; @@ -383,22 +375,22 @@ private LfSense LcmSenseToLfSense(ILexSense lcmSense) ILgWritingSystem AnalysisWritingSystem = ServiceLocator.LanguageProject.DefaultAnalysisWritingSystem; lfSense.Guid = lcmSense.Guid; - lfSense.Gloss = ToMultiText(lcmSense.Gloss); - lfSense.Definition = ToMultiText(lcmSense.Definition); + lfSense.Gloss = ToMultiText(lcmSense.Gloss, lcmSense.Guid); + lfSense.Definition = ToMultiText(lcmSense.Definition, lcmSense.Guid); // Fields below in alphabetical order by ILexSense property, except for Guid, Gloss and Definition lfSense.AcademicDomains = ToStringArrayField(AcademicDomainListCode, lcmSense.DomainTypesRC); lfSense.AnthropologyCategories = ToStringArrayField(AnthroCodeListCode, lcmSense.AnthroCodesRC); - lfSense.AnthropologyNote = ToMultiText(lcmSense.AnthroNote); - lfSense.DiscourseNote = ToMultiText(lcmSense.DiscourseNote); - lfSense.EncyclopedicNote = ToMultiText(lcmSense.EncyclopedicInfo); + lfSense.AnthropologyNote = ToMultiText(lcmSense.AnthroNote, lcmSense.Guid); + lfSense.DiscourseNote = ToMultiText(lcmSense.DiscourseNote, lcmSense.Guid); + lfSense.EncyclopedicNote = ToMultiText(lcmSense.EncyclopedicInfo, lcmSense.Guid); if (lcmSense.ExamplesOS != null) { lfSense.Examples = new List(lcmSense.ExamplesOS.Select(LcmExampleToLfExample)); } - lfSense.GeneralNote = ToMultiText(lcmSense.GeneralNote); - lfSense.GrammarNote = ToMultiText(lcmSense.GrammarNote); + lfSense.GeneralNote = ToMultiText(lcmSense.GeneralNote, lcmSense.Guid); + lfSense.GrammarNote = ToMultiText(lcmSense.GrammarNote, lcmSense.Guid); // LcmSense.LIFTid not mapped (changed 2019-10 by RM since the LiftId in LF is not useful: see LF-378) if (lcmSense.MorphoSyntaxAnalysisRA != null) { @@ -416,7 +408,7 @@ private LfSense LcmSenseToLfSense(ILexSense lcmSense) lfSense.SecondaryPartOfSpeech = ToStringField(GrammarListCode, secondaryPos); // It's fine if secondaryPos is still null here } } - lfSense.PhonologyNote = ToMultiText(lcmSense.PhonologyNote); + lfSense.PhonologyNote = ToMultiText(lcmSense.PhonologyNote, lcmSense.Guid); if (lcmSense.PicturesOS != null) { lfSense.Pictures = new List(lcmSense.PicturesOS.Select(LcmPictureToLfPicture)); @@ -426,23 +418,23 @@ private LfSense LcmSenseToLfSense(ILexSense lcmSense) //foreach (var LcmPic in lcmSense.PicturesOS) // lfSense.Pictures.Add(LcmPictureToLfPicture(LcmPic)); } - lfSense.SenseBibliography = ToMultiText(lcmSense.Bibliography); - lfSense.SenseRestrictions = ToMultiText(lcmSense.Restrictions); + lfSense.SenseBibliography = ToMultiText(lcmSense.Bibliography, lcmSense.Guid); + lfSense.SenseRestrictions = ToMultiText(lcmSense.Restrictions, lcmSense.Guid); if (lcmSense.ReferringReversalIndexEntries != null) { - IEnumerable reversalEntries = lcmSense.ReferringReversalIndexEntries.Select(lcmReversalEntry => lcmReversalEntry.LongName); - lfSense.ReversalEntries = LfStringArrayField.FromStrings(reversalEntries); + var reversalEntries = lcmSense.ReferringReversalIndexEntries.Select(e => LfStringField.CreateFrom(e.LongName, lcmSense.Guid)); + lfSense.ReversalEntries = LfStringArrayField.CreateFrom(reversalEntries); } - lfSense.ScientificName = LfMultiText.FromSingleITsString(lcmSense.ScientificName, ServiceLocator.WritingSystemFactory); + lfSense.ScientificName = LfMultiText.FromSingleITsString(lcmSense.ScientificName, lcmSense.Guid, ServiceLocator.WritingSystemFactory); lfSense.SemanticDomain = ToStringArrayField(SemDomListCode, lcmSense.SemanticDomainsRC); - lfSense.SemanticsNote = ToMultiText(lcmSense.SemanticsNote); + lfSense.SemanticsNote = ToMultiText(lcmSense.SemanticsNote, lcmSense.Guid); // lcmSense.SensesOS; // Not mapped because LF doesn't handle subsenses. TODO: When LF handles subsenses, map this one. lfSense.SenseType = ToStringField(SenseTypeListCode, lcmSense.SenseTypeRA); - lfSense.SociolinguisticsNote = ToMultiText(lcmSense.SocioLinguisticsNote); + lfSense.SociolinguisticsNote = ToMultiText(lcmSense.SocioLinguisticsNote, lcmSense.Guid); if (lcmSense.Source != null) { - lfSense.Source = LfMultiText.FromSingleITsString(lcmSense.Source, ServiceLocator.WritingSystemFactory); + lfSense.Source = LfMultiText.FromSingleITsString(lcmSense.Source, lcmSense.Guid, ServiceLocator.WritingSystemFactory); } lfSense.Status = ToStringArrayField(StatusListCode, lcmSense.StatusRA); lfSense.Usages = ToStringArrayField(UsageTypeListCode, lcmSense.UsageTypesRC); @@ -530,8 +522,8 @@ private LfExample LcmExampleToLfExample(ILexExampleSentence LcmExample) ILgWritingSystem VernacularWritingSystem = ServiceLocator.LanguageProject.DefaultVernacularWritingSystem; lfExample.Guid = LcmExample.Guid; - lfExample.Sentence = ToMultiText(LcmExample.Example); - lfExample.Reference = LfMultiText.FromSingleITsString(LcmExample.Reference, ServiceLocator.WritingSystemFactory); + lfExample.Sentence = ToMultiText(LcmExample.Example, LcmExample.Guid); + lfExample.Reference = LfMultiText.FromSingleITsString(LcmExample.Reference, LcmExample.Guid, ServiceLocator.WritingSystemFactory); // ILexExampleSentence fields we currently do not convert: // LcmExample.DoNotPublishInRC; // LcmExample.LiftResidue; @@ -545,7 +537,7 @@ private LfExample LcmExampleToLfExample(ILexExampleSentence LcmExample) // TODO: Once LF improves its data model for translations, persist all of them instead of just the first. foreach (ICmTranslation translation in LcmExample.TranslationsOC.Take(1)) { - lfExample.Translation = ToMultiText(translation.Translation); + lfExample.Translation = ToMultiText(translation.Translation, LcmExample.Guid); lfExample.TranslationGuid = translation.Guid; } @@ -561,7 +553,7 @@ private LfExample LcmExampleToLfExample(ILexExampleSentence LcmExample) private LfPicture LcmPictureToLfPicture(ICmPicture LcmPicture) { var result = new LfPicture(); - result.Caption = ToMultiText(LcmPicture.Caption); + result.Caption = ToMultiText(LcmPicture.Caption, LcmPicture.Guid); if ((LcmPicture.PictureFileRA != null) && (!string.IsNullOrEmpty(LcmPicture.PictureFileRA.InternalPath))) { result.FileName = LcmPictureFilenameToLfPictureFilename(LcmPicture.PictureFileRA.InternalPath); diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoOptionList.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoOptionList.cs index 4196131d..14b45619 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoOptionList.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoOptionList.cs @@ -15,8 +15,6 @@ public class ConvertLcmToMongoOptionList protected int _wsForKeys; protected LfOptionList _lfOptionList; protected Dictionary _lfOptionListItemByGuid; - protected Dictionary _lfOptionListItemByStrKey; - protected Dictionary _lfOptionListItemKeyByGuid; protected ILogger _logger; protected ILgWritingSystemFactory _wsf; @@ -35,7 +33,10 @@ public ConvertLcmToMongoOptionList(LfOptionList lfOptionList, int wsForKeys, str if (lfOptionList == null) lfOptionList = MakeEmptyOptionList(listCode); _lfOptionList = lfOptionList; - UpdateOptionListItemDictionaries(_lfOptionList); + + _lfOptionListItemByGuid = lfOptionList.Items + .Where(item => item.Guid != null) + .ToDictionary(item => item.Guid.Value, item => item); } public static string LcmOptionListName(string listCode) @@ -68,7 +69,6 @@ public virtual LfOptionList PrepareOptionListUpdate(ICmPossibilityList lcmOption optionListDiffersFromOriginal = true; } _lfOptionListItemByGuid[poss.Guid] = correspondingItem; - _lfOptionListItemByStrKey[correspondingItem.Key] = correspondingItem; } var lfNewOptionList = CloneOptionListWithEmptyItems(_lfOptionList); @@ -92,68 +92,13 @@ public virtual LfOptionList PrepareOptionListUpdate(ICmPossibilityList lcmOption return lfNewOptionList; } - public string LfItemKeyString(ICmPossibility lcmOptionListItem, int ws) - { - string result; - if (lcmOptionListItem == null) - return null; - - if (_lfOptionList != null) - { - if (_lfOptionListItemKeyByGuid.TryGetValue(lcmOptionListItem.Guid, out result)) - return result; - - // We shouldn't get here, because the option list SHOULD be pre-populated. - _logger.Error("Got an option list item without a corresponding LF option list item. " + - "In option list name '{0}', list code '{1}': " + - "LCM option list item '{2}' had GUID {3} but no LF option list item was found", - _lfOptionList.Name, _lfOptionList.Code, - lcmOptionListItem.AbbrAndName, lcmOptionListItem.Guid - ); - return null; - } - - if (lcmOptionListItem.Abbreviation == null || lcmOptionListItem.Abbreviation.get_String(ws) == null) - { - // Last-ditch effort - char ORC = '\ufffc'; - return lcmOptionListItem.AbbrevHierarchyString.Split(ORC).LastOrDefault(); - } - else - { - return ConvertLcmToMongoTsStrings.TextFromTsString(lcmOptionListItem.Abbreviation.get_String(ws), _wsf); - } - } - - // For multi-option lists; use like "LfStringArrayField.FromStrings(_converter.LfItemKeyStrings(PossibilityList), _wsEn)". - public IEnumerable LfItemKeyStrings(IEnumerable lcmOptionListItems, int ws) - { - foreach (ICmPossibility lcmOptionListItem in lcmOptionListItems) - yield return LfItemKeyString(lcmOptionListItem, ws); - } - protected LfOptionListItem CmPossibilityToOptionListItem(ICmPossibility pos) { var item = new LfOptionListItem(); - SetOptionListItemFromCmPossibility(item, pos, true); // Ignore the bool result since this will always modify the item + SetOptionListItemFromCmPossibility(item, pos); // Ignore the bool result since this will always modify the item return item; } - protected void UpdateOptionListItemDictionaries(LfOptionList lfOptionList) - { - _lfOptionListItemByGuid = lfOptionList.Items - .Where(item => item.Guid != null) - .ToDictionary(item => item.Guid.Value, item => item); - _lfOptionListItemByStrKey = lfOptionList.Items - .ToDictionary(item => item.Key, item => item); - _lfOptionListItemKeyByGuid = _lfOptionList.Items - .Where(item => item.Guid != null) - .ToDictionary( - item => item.Guid.GetValueOrDefault(), - item => item.Key - ); - } - protected LfOptionList MakeEmptyOptionList(string listCode) { var result = new LfOptionList(); @@ -183,22 +128,8 @@ protected LfOptionList CloneOptionListWithEmptyItems(LfOptionList original) return newList; } - protected string FindAppropriateKey(string originalKey) - { - if (originalKey == null) - originalKey = MagicStrings.UnknownString; // Can't let a null key exist, so use something non-representative - string currentTry = originalKey; - int extraNum = 0; - while (_lfOptionListItemByStrKey.ContainsKey(currentTry)) - { - extraNum++; - currentTry = originalKey + extraNum.ToString(); - } - return currentTry; - } - // Returns true if the item passed in has been modified at all, false otherwise - protected bool SetOptionListItemFromCmPossibility(LfOptionListItem item, ICmPossibility poss, bool setKey = false) + protected bool SetOptionListItemFromCmPossibility(LfOptionListItem item, ICmPossibility poss) { bool modified = false; string abbreviation = ConvertLcmToMongoTsStrings.TextFromTsString(poss.Abbreviation.BestAnalysisVernacularAlternative, _wsf); @@ -207,15 +138,6 @@ protected bool SetOptionListItemFromCmPossibility(LfOptionListItem item, ICmPoss modified = true; } item.Abbreviation = abbreviation; - if (setKey) - { - string key = FindAppropriateKey(ConvertLcmToMongoTsStrings.TextFromTsString(poss.Abbreviation.get_String(_wsForKeys), _wsf)); - if (item.Key != key) - { - modified = true; - } - item.Key = key; - } string value = ConvertLcmToMongoTsStrings.TextFromTsString(poss.Name.BestAnalysisVernacularAlternative, _wsf); if (item.Value != value) { diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs index 164bbce3..63ed0204 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs @@ -24,7 +24,8 @@ public class ConvertMongoToLcmOptionList protected ICmPossibilityList _parentList; #endif - public Dictionary PossibilitiesByKey { get; protected set; } + //todo + public Dictionary Possibilities { get; protected set; } #if false // Once we allow LanguageForge to create optionlist items with "canonical" values (parts of speech, semantic domains, etc.), uncomment this version of the constructor public ConvertMongoToLcmOptionList(IRepository possRepo, LfOptionList lfOptionList, ILogger logger, ICmPossibilityList parentList, int wsForKeys, CanonicalOptionListSource canonicalSource = null) @@ -46,37 +47,17 @@ public virtual void RebuildLookupTables(LfOptionList lfOptionList) { _lfOptionList = lfOptionList; - PossibilitiesByKey = new Dictionary(); + Possibilities = new Dictionary(); if (lfOptionList == null || lfOptionList.Items == null) return; foreach (LfOptionListItem item in lfOptionList.Items) { ICmPossibility poss = LookupByItem(item); if (poss != null) - PossibilitiesByKey[item.Key] = poss; + Possibilities[item.Guid] = poss; } } - public ICmPossibility FromStringKey(string key) - { - ICmPossibility result; - if (PossibilitiesByKey.TryGetValue(key, out result)) - return result; - if (_canonicalSource != null) - return LookupByCanonicalItem(_canonicalSource.ByKeyOrNull(key)); - #if false // Once we allow LanguageForge to create optionlist items with "canonical" values (parts of speech, semantic domains, etc.), uncomment this block to replace the "return _canonicalSource.ByKeyOrNull(key)" line above. - if (_canonicalSource != null) - { - CanonicalItem item; - if (_canonicalSource.TryGetByKey(key, out item)) - { - return CreateFromCanonicalItem(item); - } - } - #endif - return null; - } - #if false // Once we allow LanguageForge to create optionlist items with "canonical" values (parts of speech, semantic domains, etc.), uncomment this block public ICmPossibility CreateFromCanonicalItem(CanonicalItem item) { @@ -97,29 +78,29 @@ public ICmPossibility FromStringField(LfStringField keyField) { if (keyField == null) return null; - string key = keyField.ToString(); - if (string.IsNullOrEmpty(key)) - return null; - return FromStringKey(key); + if (Possibilities.TryGetValue(keyField.LcmGuid, out var result)) + return result; + if (_canonicalSource != null) + return LookupByCanonicalItem(_canonicalSource.ByKeyOrNull(keyField.Value)); + + return null; } public ICmPossibility FromStringArrayFieldWithOneCase(LfStringArrayField keyField) { if (keyField == null || keyField.Values == null || keyField.IsEmpty) return null; - return FromStringKey(keyField.Values.First()); + return Possibilities[keyField.LcmGuids.First()]; } // Used in UpdatePossibilitiesFromStringArray and UpdateInvertedPossibilitiesFromStringArray below // Generic so that they can handle lists like AnthroCodes, etc. public IEnumerable FromStringArrayField(LfStringArrayField source) { - IEnumerable keys; - if (source == null || source.Values == null) - keys = new List(); // Empty list - else - keys = source.Values.Where(value => !string.IsNullOrEmpty(value)); - return keys.Select(key => (T)FromStringKey(key)).Where(poss => poss != null); + if (source == null) return new List(); + + var result = source.LcmGuids.Select(g => (T) Possibilities[g]); + return result; } protected ICmPossibility LookupByItem(LfOptionListItem item) @@ -168,7 +149,7 @@ public void SetPossibilitiesCollection(ILcmReferenceCollection dest, IEnum // If we know of NO valid possibility keys, don't make any changes. That's because knowing of NO valid possibility keys // is FAR more likely to happen because of a bug than because we really removed an entire possibility list, and if there's // a bug, we shouldn't drop all the Lcm data for this possibility list. - if (PossibilitiesByKey.Count == 0 && _canonicalSource == null) + if (Possibilities.Count == 0 && _canonicalSource == null) return; // We have to calculate the update (which items to remove and which to add) here; ILcmReferenceCollection won't do it for us. List itemsToAdd = newItems.ToList(); diff --git a/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs b/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs index e85bcb20..b327586b 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs @@ -20,7 +20,7 @@ protected bool _ShouldSerializeLfMultiText(LfMultiText value) protected bool _ShouldSerializeLfStringArrayField(LfStringArrayField value) { - return value != null && !value.IsEmpty && value.Values.TrueForAll(s => !string.IsNullOrEmpty(s)); + return value != null && !value.IsEmpty && value.Values.ToList().TrueForAll(s => !string.IsNullOrEmpty(s)); } protected bool _ShouldSerializeLfStringField(LfStringField value) diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 96dd29dd..287d0e44 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -14,7 +14,7 @@ public class LfMultiText : Dictionary // Note: NOT derive { public bool IsEmpty { get { return (Count <= 0) || (this.All(kv => kv.Value == null || kv.Value.IsEmpty)); } } - public static LfMultiText FromLcmMultiString(IMultiAccessorBase other, ILgWritingSystemFactory wsManager) + public static LfMultiText FromLcmMultiString(IMultiAccessorBase other, Guid lcmId, ILgWritingSystemFactory wsManager) { LfMultiText newInstance = new LfMultiText(); foreach (int wsid in other.AvailableWritingSystemIds) @@ -22,34 +22,34 @@ public static LfMultiText FromLcmMultiString(IMultiAccessorBase other, ILgWritin string wsstr = wsManager.GetStrFromWs(wsid); ITsString value = other.get_String(wsid); string text = LfMerge.Core.DataConverters.ConvertLcmToMongoTsStrings.TextFromTsString(value, wsManager); - LfStringField field = LfStringField.FromString(text); + LfStringField field = LfStringField.CreateFrom(text, lcmId); if (field != null) newInstance.Add(wsstr, field); } return newInstance; } - public static LfMultiText FromSingleStringMapping(string key, string value) + public static LfMultiText FromSingleStringMapping(string key, string value, Guid lcmId) { - LfStringField field = LfStringField.FromString(value); + LfStringField field = LfStringField.CreateFrom(value, lcmId); if (field == null) return null; return new LfMultiText { { key, field } }; } - public static LfMultiText FromSingleITsString(ITsString value, ILgWritingSystemFactory wsManager) + public static LfMultiText FromSingleITsString(ITsString value, Guid lcmId, ILgWritingSystemFactory wsManager) { if (value == null || value.Text == null) return null; int wsId = value.get_WritingSystem(0); string wsStr = wsManager.GetStrFromWs(wsId); string text = LfMerge.Core.DataConverters.ConvertLcmToMongoTsStrings.TextFromTsString(value, wsManager); - LfStringField field = LfStringField.FromString(text); + LfStringField field = LfStringField.CreateFrom(text, lcmId); if (field == null) return null; return new LfMultiText { { wsStr, field } }; } - public static LfMultiText FromMultiITsString(ITsMultiString value, ILgWritingSystemFactory wsManager) + public static LfMultiText FromMultiITsString(ITsMultiString value, Guid lcmId, ILgWritingSystemFactory wsManager) { if (value == null || value.StringCount == 0) return null; LfMultiText mt = new LfMultiText(); @@ -61,7 +61,7 @@ public static LfMultiText FromMultiITsString(ITsMultiString value, ILgWritingSys if (!string.IsNullOrEmpty(wsStr)) { string valueStr = LfMerge.Core.DataConverters.ConvertLcmToMongoTsStrings.TextFromTsString(tss, wsManager); - LfStringField field = LfStringField.FromString(valueStr); + LfStringField field = LfStringField.CreateFrom(valueStr, lcmId); if (field != null) mt.Add(wsStr, field); } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs b/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs index dc64800d..429550db 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs @@ -6,11 +6,31 @@ namespace LfMerge.Core.LanguageForge.Model { - public class LfOptionListItem : IHasNullableGuid - { + public class LfOptionListItem : IHasNullableGuid + { + private string key; + [BsonRepresentation(BsonType.String)] public Guid? Guid { get; set; } - public string Key { get; set; } + public string Key + { + get => key; + set + { + if (System.Guid.TryParse(value, out var guid)) + { + key = guid.ToString(); + } + else if (Guid.HasValue) + { + key = Guid.Value.ToString(); + } + else + { + throw new ApplicationException("Cannot set Key property to non-GUID value " + value); + } + } + } public string Value { get; set; } public string Abbreviation { get; set; } } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs b/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs index d7d8f5aa..4e901293 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs @@ -3,29 +3,25 @@ using System.Collections.Generic; using System.Linq; +using System; namespace LfMerge.Core.LanguageForge.Model { public class LfStringArrayField : LfFieldBase { - public List Values { get; set; } + private IList _values = new List(); + public IEnumerable LcmGuids { get { return _values.Select(v => v.LcmGuid); } } + public List Values { get; set; } public bool IsEmpty { get { return Values.Count <= 0; } } - public LfStringArrayField() - { - Values = new List(); - } - - public static LfStringArrayField FromStrings(IEnumerable source) - { - return new LfStringArrayField { Values = new List(source.Where(s => s != null)) }; - } + private LfStringArrayField() { } - public static LfStringArrayField FromSingleString(string source) + public static LfStringArrayField CreateFrom(IEnumerable source) { - if (source == null) return null; - return new LfStringArrayField { Values = new List { source } }; + if (source == null) + throw new ApplicationException("Tried to create LfStringArrayField with no source."); + return new LfStringArrayField { _values = source.Where(f => f != null).ToList() }; } } } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs b/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs index 3bf796b6..baddcffd 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs @@ -2,38 +2,30 @@ // This software is licensed under the MIT license (http://opensource.org/licenses/MIT) using System; using System.Collections.Generic; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; namespace LfMerge.Core.LanguageForge.Model { public class LfStringField : LfFieldBase { - public string Value { get; set; } + public string Value { get; private set; } - [BsonRepresentation(BsonType.String)] // Yes, this works with List. Very nice. - public List Guids { get; set; } // Used only for custom MultiPara fields. Empty or missing otherwise. - public bool ShouldSerializeGuids() { return (Guids != null) && (Guids.Count > 0); } + public Guid LcmGuid { get; private set; } public bool IsEmpty { get { return String.IsNullOrEmpty(Value); } } public override string ToString() { return Value; - // return string.Format("[LfStringField: Value={0}]", Value); } - public static LfStringField FromString(string source) + public static LfStringField CreateFrom(string source, Guid lcmId) { - if (source == null) + if (source == null || lcmId == null) return null; - return new LfStringField { Value = source, Guids = new List() }; + return new LfStringField { Value = source, LcmGuid = lcmId }; } - public LfStringField() - { - Guids = new List(); - } + private LfStringField() { } public Dictionary AsDictionary() { From d1977042e8ddce20aa2c4ff650eb836e38ae3697 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Wed, 9 Nov 2022 15:11:50 -0600 Subject: [PATCH 4/6] Made LfStringField and LfStringArrayField GUID-less again. I realized the actual problem was that the LfSense model was incorrectly representing the option list items as String*Fields. Now LfSense uses LfOptionListItem's and List to better represent LF. Note that Status went from LfStringArrayField -> LfOptionListItem, instead of a list. Seems like at one point LF was allowing multiple status values? Currently it is only allowing one, so I made that switch. I also cleaned up the ConvertMongoToLcmOptionList class some. I removed the now unneeded ability to look up LCM items from StringFields. The API now exposes LookupByItem(LfOptionListItem), which looks through the Possibilities, then the _possRepo, then the CanonicalItem's, similar to what the FromStringField was doing. The new API is cleaner and less confusing. Also added some default value for Key on LfOptionListItem. If it is requested before it's been set, it will use the GUID. Also gave it an IsEmpty method to integrate with serialization. --- .../Actions/SynchronizeActionTests.cs | 8 +- src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs | 28 ++--- .../Lcm/TransferMongoToLcmActionTests.cs | 26 ++--- .../ConvertLcmToMongoCustomField.cs | 10 +- .../ConvertLcmToMongoLexicon.cs | 106 ++++++++++-------- .../ConvertMongoToLcmLexicon.cs | 10 +- .../ConvertMongoToLcmOptionList.cs | 51 +++------ .../LanguageForge/Model/LfFieldBase.cs | 5 + .../LanguageForge/Model/LfLexEntry.cs | 4 +- .../LanguageForge/Model/LfMultiText.cs | 16 +-- .../LanguageForge/Model/LfOptionListItem.cs | 7 +- .../LanguageForge/Model/LfSense.cs | 32 +++--- .../LanguageForge/Model/LfStringArrayField.cs | 1 - .../LanguageForge/Model/LfStringField.cs | 8 +- 14 files changed, 151 insertions(+), 161 deletions(-) diff --git a/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs b/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs index 88eb1c52..d7d662ba 100644 --- a/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs +++ b/src/LfMerge.Core.Tests/Actions/SynchronizeActionTests.cs @@ -164,7 +164,7 @@ public void SynchronizeAction_LFDataChanged_GlossChanged() IEnumerable originalMongoData = _mongoConnection.GetLfLexEntries(); LfLexEntry lfEntry = originalMongoData.First(e => e.Guid == _testEntryGuid); string unchangedGloss = lfEntry.Senses[0].Gloss["en"].Value; - var lfChangedGloss = LfStringField.CreateFrom(unchangedGloss + " - changed in LF", new Guid()); + var lfChangedGloss = LfStringField.CreateFrom(unchangedGloss + " - changed in LF"); lfEntry.Senses[0].Gloss["en"] = lfChangedGloss; _mongoConnection.UpdateRecord(_lfProject, lfEntry); @@ -246,7 +246,7 @@ public void SynchronizeAction_LFDataChangedLDDataChanged_LFWins() string unchangedGloss = lfEntry.Senses[0].Gloss["en"].Value; string fwChangedGloss = unchangedGloss + " - changed in FW"; - var lfChangedGloss = LfStringField.CreateFrom(unchangedGloss + " - changed in LF", new Guid()); + var lfChangedGloss = LfStringField.CreateFrom(unchangedGloss + " - changed in LF"); lfEntry.Senses[0].Gloss["en"] = lfChangedGloss; lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _mongoConnection.UpdateRecord(_lfProject, lfEntry); @@ -436,7 +436,7 @@ public void SynchronizeAction_LFDataChangedLDEntryDeleted_LFWins() const string lfCreatedGloss = "new English gloss - added in LF"; const string fwChangedGloss = "English gloss - changed in FW"; // LF adds a gloss to the entry that LD is deleting - lfEntry.Senses[0].Gloss = LfMultiText.FromSingleStringMapping("en", lfCreatedGloss, new Guid()); + lfEntry.Senses[0].Gloss = LfMultiText.FromSingleStringMapping("en", lfCreatedGloss); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _mongoConnection.UpdateRecord(_lfProject, lfEntry); @@ -480,7 +480,7 @@ public void SynchronizeAction_LFDataChangedLDOtherDataChanged_ModifiedDateUpdate var originalLfDateModified = lfEntry.DateModified; var originalLfAuthorInfoModifiedDate = lfEntry.AuthorInfo.ModifiedDate; - lfEntry.Note = LfMultiText.FromSingleStringMapping("en", "A note from LF", new Guid()); + lfEntry.Note = LfMultiText.FromSingleStringMapping("en", "A note from LF"); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _mongoConnection.UpdateRecord(_lfProject, lfEntry); diff --git a/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs b/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs index 6633c619..35926788 100644 --- a/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/RoundTripTests.cs @@ -181,7 +181,7 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInEntries() string vernacularWS = cache.ServiceLocator.WritingSystemManager.GetStrFromWs(cache.DefaultVernWs); string originalLexeme = originalLfEntry.Lexeme[vernacularWS].Value; string changedLexeme = "Changed lexeme for this test"; - var originalValue = LfStringField.CreateFrom(changedLexeme, new Guid()); + var originalValue = LfStringField.CreateFrom(changedLexeme); originalLfEntry.Lexeme[vernacularWS] = originalValue; originalLfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalLfEntry); @@ -193,7 +193,7 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInEntries() // Exercise SutMongoToLcm.Run(lfProject); string changedLexemeDuringUpdate = "This value should be overwritten by LcmToMongo"; - originalLfEntry.Lexeme[vernacularWS] = LfStringField.CreateFrom(changedLexemeDuringUpdate, new Guid()); + originalLfEntry.Lexeme[vernacularWS] = LfStringField.CreateFrom(changedLexemeDuringUpdate); originalLfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(originalLfEntry); SutLcmToMongo.Run(lfProject); @@ -269,8 +269,8 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInSenses() var originalSense0Definition = originalEntry.Senses[0].Definition["en"]; var originalSense1Definition = originalEntry.Senses[1].Definition["en"]; - var changedSense0Definition = LfStringField.CreateFrom("Changed sense0 definition for this test", new Guid()); - var changedSense1Definition = LfStringField.CreateFrom("Changed sense1 definition for this test", new Guid()); + var changedSense0Definition = LfStringField.CreateFrom("Changed sense0 definition for this test"); + var changedSense1Definition = LfStringField.CreateFrom("Changed sense1 definition for this test"); originalEntry.Senses[0].Definition["en"] = changedSense0Definition; originalEntry.Senses[1].Definition["en"] = changedSense1Definition; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -278,7 +278,7 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInSenses() // Exercise SutMongoToLcm.Run(lfProject); - var changedDefinitionDuringUpdate = LfStringField.CreateFrom("This value should be overwritten by LcmToMongo", new Guid()); + var changedDefinitionDuringUpdate = LfStringField.CreateFrom("This value should be overwritten by LcmToMongo"); originalEntry.Senses[0].Definition["en"] = changedDefinitionDuringUpdate; originalEntry.Senses[1].Definition["en"] = changedDefinitionDuringUpdate; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -381,8 +381,8 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInExample() var originalSense0Example0Translation = originalEntry.Senses[0].Examples[0].Translation["en"]; var originalSense0Example1Translation = originalEntry.Senses[0].Examples[1].Translation["en"]; - var changedSense0Example0Translation = LfStringField.CreateFrom("Changed sense0 example0 sentence for this test", new Guid()); - var changedSense0Example1Translation = LfStringField.CreateFrom("Changed sense0 example1 sentence for this test", new Guid()); + var changedSense0Example0Translation = LfStringField.CreateFrom("Changed sense0 example0 sentence for this test"); + var changedSense0Example1Translation = LfStringField.CreateFrom("Changed sense0 example1 sentence for this test"); originalEntry.Senses[0].Examples[0].Translation["en"] = changedSense0Example0Translation; originalEntry.Senses[0].Examples[1].Translation["en"] = changedSense0Example1Translation; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -390,7 +390,7 @@ public void RoundTrip_LcmToMongoToLcmToMongo_ShouldKeepModifiedValuesInExample() // Exercise SutMongoToLcm.Run(lfProject); - var changedTranslationDuringUpdate = LfStringField.CreateFrom("This value should be overwritten by LcmToMongo", new Guid()); + var changedTranslationDuringUpdate = LfStringField.CreateFrom("This value should be overwritten by LcmToMongo"); originalEntry.Senses[0].Examples[0].Translation["en"] = changedTranslationDuringUpdate; originalEntry.Senses[0].Examples[1].Translation["en"] = changedTranslationDuringUpdate; originalEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -473,7 +473,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewEntry() newEntry.Guid = Guid.NewGuid(); string vernacularWS = lfProject.FieldWorksProject.Cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = new DateTime(); newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -553,8 +553,8 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewSense() Assert.That(lfEntry.Senses.Count, Is.EqualTo(2)); LfSense newSense = new LfSense(); newSense.Guid = Guid.NewGuid(); - newSense.Definition = LfMultiText.FromSingleStringMapping(vernacularWS, newDefinition, newSense.Guid.Value); - newSense.PartOfSpeech = LfStringField.CreateFrom(newPartOfSpeech, newSense.Guid.Value); + newSense.Definition = LfMultiText.FromSingleStringMapping(vernacularWS, newDefinition); + newSense.PartOfSpeech = new LfOptionListItem { Value = newPartOfSpeech }; lfEntry.Senses.Add(newSense); Assert.That(lfEntry.Senses.Count, Is.EqualTo(3)); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -643,8 +643,8 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewExample() Assert.That(lfSense.Examples.Count, Is.EqualTo(2)); LfExample newExample = new LfExample(); newExample.Guid = Guid.NewGuid(); - newExample.Sentence = LfMultiText.FromSingleStringMapping(vernacularWS, newSentence, newExample.Guid.Value); - newExample.Translation = LfMultiText.FromSingleStringMapping(vernacularWS, newTranslation, newExample.Guid.Value); + newExample.Sentence = LfMultiText.FromSingleStringMapping(vernacularWS, newSentence); + newExample.Translation = LfMultiText.FromSingleStringMapping(vernacularWS, newTranslation); lfSense.Examples.Add(newExample); Assert.That(lfSense.Examples.Count, Is.EqualTo(3)); lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; @@ -737,7 +737,7 @@ public void RoundTrip_MongoToLcmToMongo_ShouldAddAndDeleteNewPicture() Assert.That(lfSense.Pictures.Count, Is.EqualTo(1)); LfPicture newPicture = new LfPicture(); newPicture.Guid = Guid.NewGuid(); - newPicture.Caption = LfMultiText.FromSingleStringMapping(vernacularWS, newCaption, newPicture.Guid.Value); + newPicture.Caption = LfMultiText.FromSingleStringMapping(vernacularWS, newCaption); newPicture.FileName = newFilename; lfSense.Pictures.Add(newPicture); Assert.That(lfSense.Pictures.Count, Is.EqualTo(2)); diff --git a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs index dde7408d..9a7a17e0 100644 --- a/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs +++ b/src/LfMerge.Core.Tests/Lcm/TransferMongoToLcmActionTests.cs @@ -96,7 +96,7 @@ public void Action_ChangedWithSampleData_ShouldUpdatePictures() Is.EqualTo(expectedExternalFileName)); LfMultiText expectedNewCaption = LfMultiText.FromLcmMultiString( - entry.SensesOS[0].PicturesOS[0].Caption, entry.Guid, cache.ServiceLocator.WritingSystemFactory); + entry.SensesOS[0].PicturesOS[0].Caption, cache.ServiceLocator.WritingSystemFactory); int expectedNumOfNewCaptions = expectedNewCaption.Count(); Assert.That(expectedNumOfNewCaptions, Is.EqualTo(2)); string expectedNewVernacularCaption = expectedNewCaption["qaa-x-kal"].Value; @@ -207,7 +207,7 @@ public void Action_WithOneNewEntry_ShouldCountOneAdded() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -237,7 +237,7 @@ public void Action_WithOneModifiedEntry_ShouldCountOneModified() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -289,7 +289,7 @@ public void Action_WithTwoNewEntries_ShouldCountTwoAdded() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -298,7 +298,7 @@ public void Action_WithTwoNewEntries_ShouldCountTwoAdded() LfLexEntry newEntry2 = new LfLexEntry(); newEntry2.Guid = Guid.NewGuid(); string newLexeme2 = "new lexeme #2 for this test"; - newEntry2.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme2, newEntry2.Guid.Value); + newEntry2.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme2); newEntry2.AuthorInfo = new LfAuthorInfo(); newEntry2.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry2.AuthorInfo.ModifiedDate = newEntry2.AuthorInfo.CreatedDate; @@ -328,7 +328,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -336,7 +336,7 @@ public void Action_WithTwoModifiedEntries_ShouldCountTwoModified() Guid kenGuid = Guid.Parse(KenEntryGuidStr); LfLexEntry kenEntry = _conn.GetLfLexEntryByGuid(kenGuid); string changedLexeme2 = "modified lexeme #2 for this test"; - kenEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2, kenGuid); + kenEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2); kenEntry.AuthorInfo = new LfAuthorInfo(); kenEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(kenEntry); @@ -392,7 +392,7 @@ public void Action_WithOneNewEntry_ShouldNotCountThatNewEntryOnSecondRun() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -433,7 +433,7 @@ public void Action_WithOneModifiedEntry_ShouldNotCountThatModifiedEntryOnSecondR LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -507,7 +507,7 @@ public void Action_RunTwiceWithOneNewEntryEachTime_ShouldCountTwoAddedInTotal() LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string newLexeme = "new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -528,7 +528,7 @@ public void Action_RunTwiceWithOneNewEntryEachTime_ShouldCountTwoAddedInTotal() newEntry = new LfLexEntry(); newEntry.Guid = Guid.NewGuid(); newLexeme = "second new lexeme for this test"; - newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme, newEntry.Guid.Value); + newEntry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, newLexeme); newEntry.AuthorInfo = new LfAuthorInfo(); newEntry.AuthorInfo.CreatedDate = DateTime.UtcNow; newEntry.AuthorInfo.ModifiedDate = newEntry.AuthorInfo.CreatedDate; @@ -560,7 +560,7 @@ public void Action_RunTwiceWithTheSameEntryModifiedEachTime_ShouldCountTwoModifi LcmCache cache = lfProj.FieldWorksProject.Cache; string vernacularWS = cache.LanguageProject.DefaultVernacularWritingSystem.Id; string changedLexeme = "modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme, entryGuid); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); @@ -578,7 +578,7 @@ public void Action_RunTwiceWithTheSameEntryModifiedEachTime_ShouldCountTwoModifi // Setup second run string changedLexeme2 = "second modified lexeme for this test"; - entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2, entryGuid); + entry.Lexeme = LfMultiText.FromSingleStringMapping(vernacularWS, changedLexeme2); entry.AuthorInfo = new LfAuthorInfo(); entry.AuthorInfo.ModifiedDate = DateTime.UtcNow; _conn.UpdateMockLfLexEntry(entry); diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs index 6adcf34e..4fde0c53 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoCustomField.cs @@ -326,8 +326,6 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp CellarPropertyType LcmFieldType = (CellarPropertyType)LcmMetaData.GetFieldType(flid); var dataGuids = new List(); - var guid = data.get_GuidProp(hvo, flid); - // Valid field types in Lcm are GenDate, Integer, String, OwningAtomic, ReferenceAtomic, and ReferenceCollection, so that's all we implement. switch (LcmFieldType) { @@ -337,7 +335,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp // LF wants single-string fields in the format { "ws": { "value": "contents" } } fieldValue = String.IsNullOrEmpty(genDateStr) ? null : LfMultiText.FromSingleStringMapping( - MagicStrings.LanguageCodeForGenDateFields, genDateStr, guid).AsBsonDocument(); + MagicStrings.LanguageCodeForGenDateFields, genDateStr).AsBsonDocument(); break; // When parsing, will use GenDate.TryParse(str, out genDate) @@ -348,7 +346,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp else // LF wants single-string fields in the format { "ws": { "value": "contents" } } fieldValue = LfMultiText.FromSingleStringMapping( - MagicStrings.LanguageCodeForIntFields, fieldValue.AsInt32.ToString(), guid).AsBsonDocument(); + MagicStrings.LanguageCodeForIntFields, fieldValue.AsInt32.ToString()).AsBsonDocument(); break; case CellarPropertyType.OwningAtomic: @@ -365,7 +363,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp case CellarPropertyType.MultiUnicode: ITsMultiString tss = data.get_MultiStringProp(hvo, flid); if (tss != null && tss.StringCount > 0) - fieldValue = LfMultiText.FromMultiITsString(tss, guid, servLoc.WritingSystemManager).AsBsonDocument(); + fieldValue = LfMultiText.FromMultiITsString(tss, servLoc.WritingSystemManager).AsBsonDocument(); break; case CellarPropertyType.OwningCollection: case CellarPropertyType.OwningSequence: @@ -387,7 +385,7 @@ private BsonDocument GetCustomFieldData(int hvo, int flid, string fieldSourceTyp if (iTsValue == null || String.IsNullOrEmpty(iTsValue.Text)) fieldValue = null; else - fieldValue = LfMultiText.FromSingleITsString(iTsValue, guid, servLoc.WritingSystemManager).AsBsonDocument(); + fieldValue = LfMultiText.FromSingleITsString(iTsValue, servLoc.WritingSystemManager).AsBsonDocument(); break; default: fieldValue = null; diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index a43010f6..89adae69 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -187,16 +187,16 @@ private T GetInstance() where T : class return ServiceLocator.GetInstance(); } - private LfMultiText ToMultiText(IMultiAccessorBase LcmMultiString, Guid lcmGuid) + private LfMultiText ToMultiText(IMultiAccessorBase LcmMultiString) { if (LcmMultiString == null) return null; - return LfMultiText.FromLcmMultiString(LcmMultiString, lcmGuid, ServiceLocator.WritingSystemManager); + return LfMultiText.FromLcmMultiString(LcmMultiString, ServiceLocator.WritingSystemManager); } private LfStringField ToStringField(string listCode, ICmPossibility LcmPoss) { var abbreviation = ConvertLcmToMongoTsStrings.TextFromTsString(LcmPoss.Abbreviation.get_String(_wsEn), ServiceLocator.WritingSystemFactory); - return LfStringField.CreateFrom(abbreviation, LcmPoss.Guid); + return LfStringField.CreateFrom(abbreviation); } private LfStringArrayField ToStringArrayField(string listCode, IEnumerable LcmPossCollection) @@ -205,10 +205,20 @@ private LfStringArrayField ToStringArrayField(string listCode, IEnumerable() { ToStringField(listCode, LcmPoss) }); + var abbreviation = ConvertLcmToMongoTsStrings.TextFromTsString(LcmPoss.Abbreviation.get_String(_wsEn), ServiceLocator.WritingSystemFactory); + var ret = new LfOptionListItem(); + ret.Guid = LcmPoss.Guid; + ret.Key = LcmPoss.Guid.ToString(); + ret.Value = abbreviation; + ret.Abbreviation = abbreviation; + return ret; + } + + private List ToOptionListItems(string listCode, IEnumerable LcmPossCollection) + { + return LcmPossCollection.Select(p => ToOptionListItem(listCode, p)).ToList(); } /// @@ -229,7 +239,7 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) if (LcmLexeme == null) lfEntry.Lexeme = null; else - lfEntry.Lexeme = ToMultiText(LcmLexeme.Form, LcmLexeme.Guid); + lfEntry.Lexeme = ToMultiText(LcmLexeme.Form); // Other fields of LcmLexeme (AllomorphEnvironments, LiftResidue, MorphTypeRA, etc.) not mapped // Fields below in alphabetical order by ILexSense property, except for Lexeme @@ -237,11 +247,11 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) { // Do nothing; LanguageForge doesn't currently handle allomorphs, so we don't convert them } - lfEntry.EntryBibliography = ToMultiText(LcmEntry.Bibliography, LcmEntry.Guid); + lfEntry.EntryBibliography = ToMultiText(LcmEntry.Bibliography); // TODO: Consider whether to use LcmEntry.CitationFormWithAffixType instead // (which would produce "-s" instead of "s" for the English plural suffix, for instance) - lfEntry.CitationForm = ToMultiText(LcmEntry.CitationForm, LcmEntry.Guid); - lfEntry.Note = ToMultiText(LcmEntry.Comment, LcmEntry.Guid); + lfEntry.CitationForm = ToMultiText(LcmEntry.CitationForm); + lfEntry.Note = ToMultiText(LcmEntry.Comment); // DateModified and DateCreated can be confusing, because LF and Lcm are doing two different // things with them. In Lcm, there is just one DateModified and one DateCreated; simple. But @@ -280,15 +290,15 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) #endif if (LcmEtymology != null) { - lfEntry.Etymology = ToMultiText(LcmEtymology.Form, LcmEtymology.Guid); - lfEntry.EtymologyComment = ToMultiText(LcmEtymology.Comment, LcmEtymology.Guid); - lfEntry.EtymologyGloss = ToMultiText(LcmEtymology.Gloss, LcmEtymology.Guid); - lfEntry.EtymologySource = ToMultiText(LcmEtymology.LanguageNotes, LcmEtymology.Guid); + lfEntry.Etymology = ToMultiText(LcmEtymology.Form); + lfEntry.EtymologyComment = ToMultiText(LcmEtymology.Comment); + lfEntry.EtymologyGloss = ToMultiText(LcmEtymology.Gloss); + lfEntry.EtymologySource = ToMultiText(LcmEtymology.LanguageNotes); // LcmEtymology.LiftResidue not mapped } lfEntry.Guid = LcmEntry.Guid; // LcmEntry.LIFTid not mapped (changed 2019-10 by RM since the LiftId in LF is not useful: see LF-378) - lfEntry.LiteralMeaning = ToMultiText(LcmEntry.LiteralMeaning, LcmEntry.Guid); + lfEntry.LiteralMeaning = ToMultiText(LcmEntry.LiteralMeaning); if (LcmEntry.PrimaryMorphType != null) { lfEntry.MorphologyType = LcmEntry.PrimaryMorphType.NameHierarchyString; } @@ -297,17 +307,17 @@ private LfLexEntry LcmLexEntryToLfLexEntry(ILexEntry LcmEntry) if (LcmEntry.PronunciationsOS.Count > 0) { ILexPronunciation LcmPronunciation = LcmEntry.PronunciationsOS.First(); - lfEntry.Pronunciation = ToMultiText(LcmPronunciation.Form, LcmPronunciation.Guid); - lfEntry.CvPattern = LfMultiText.FromSingleITsString(LcmPronunciation.CVPattern, LcmPronunciation.Guid, ServiceLocator.WritingSystemFactory); - lfEntry.Tone = LfMultiText.FromSingleITsString(LcmPronunciation.Tone, LcmPronunciation.Guid, ServiceLocator.WritingSystemFactory); + lfEntry.Pronunciation = ToMultiText(LcmPronunciation.Form); + lfEntry.CvPattern = LfMultiText.FromSingleITsString(LcmPronunciation.CVPattern, ServiceLocator.WritingSystemFactory); + lfEntry.Tone = LfMultiText.FromSingleITsString(LcmPronunciation.Tone, ServiceLocator.WritingSystemFactory); // TODO: Map LcmPronunciation.MediaFilesOS properly (converting video to sound files if necessary) - lfEntry.Location = ToStringField(LocationListCode, LcmPronunciation.LocationRA); + lfEntry.Location = ToOptionListItem(LocationListCode, LcmPronunciation.LocationRA); } - lfEntry.EntryRestrictions = ToMultiText(LcmEntry.Restrictions, LcmEntry.Guid); + lfEntry.EntryRestrictions = ToMultiText(LcmEntry.Restrictions); if (lfEntry.Senses == null) // Shouldn't happen, but let's be careful lfEntry.Senses = new List(); lfEntry.Senses.AddRange(LcmEntry.SensesOS.Select(LcmSenseToLfSense)); - lfEntry.SummaryDefinition = ToMultiText(LcmEntry.SummaryDefinition, LcmEntry.Guid); + lfEntry.SummaryDefinition = ToMultiText(LcmEntry.SummaryDefinition); BsonDocument customFieldsAndGuids = _convertCustomField.GetCustomFieldsForThisCmObject(LcmEntry, "entry", ListConverters); BsonDocument customFieldsBson = customFieldsAndGuids["customFields"].AsBsonDocument; @@ -375,22 +385,22 @@ private LfSense LcmSenseToLfSense(ILexSense lcmSense) ILgWritingSystem AnalysisWritingSystem = ServiceLocator.LanguageProject.DefaultAnalysisWritingSystem; lfSense.Guid = lcmSense.Guid; - lfSense.Gloss = ToMultiText(lcmSense.Gloss, lcmSense.Guid); - lfSense.Definition = ToMultiText(lcmSense.Definition, lcmSense.Guid); + lfSense.Gloss = ToMultiText(lcmSense.Gloss); + lfSense.Definition = ToMultiText(lcmSense.Definition); // Fields below in alphabetical order by ILexSense property, except for Guid, Gloss and Definition - lfSense.AcademicDomains = ToStringArrayField(AcademicDomainListCode, lcmSense.DomainTypesRC); - lfSense.AnthropologyCategories = ToStringArrayField(AnthroCodeListCode, lcmSense.AnthroCodesRC); - lfSense.AnthropologyNote = ToMultiText(lcmSense.AnthroNote, lcmSense.Guid); - lfSense.DiscourseNote = ToMultiText(lcmSense.DiscourseNote, lcmSense.Guid); - lfSense.EncyclopedicNote = ToMultiText(lcmSense.EncyclopedicInfo, lcmSense.Guid); + lfSense.AcademicDomains = ToOptionListItems(AcademicDomainListCode, lcmSense.DomainTypesRC); + lfSense.AnthropologyCategories = ToOptionListItems(AnthroCodeListCode, lcmSense.AnthroCodesRC); + lfSense.AnthropologyNote = ToMultiText(lcmSense.AnthroNote); + lfSense.DiscourseNote = ToMultiText(lcmSense.DiscourseNote); + lfSense.EncyclopedicNote = ToMultiText(lcmSense.EncyclopedicInfo); if (lcmSense.ExamplesOS != null) { lfSense.Examples = new List(lcmSense.ExamplesOS.Select(LcmExampleToLfExample)); } - lfSense.GeneralNote = ToMultiText(lcmSense.GeneralNote, lcmSense.Guid); - lfSense.GrammarNote = ToMultiText(lcmSense.GrammarNote, lcmSense.Guid); + lfSense.GeneralNote = ToMultiText(lcmSense.GeneralNote); + lfSense.GrammarNote = ToMultiText(lcmSense.GrammarNote); // LcmSense.LIFTid not mapped (changed 2019-10 by RM since the LiftId in LF is not useful: see LF-378) if (lcmSense.MorphoSyntaxAnalysisRA != null) { @@ -404,11 +414,11 @@ private LfSense LcmSenseToLfSense(ILexSense lcmSense) LfProject.ProjectCode); else { - lfSense.PartOfSpeech = ToStringField(GrammarListCode, pos); - lfSense.SecondaryPartOfSpeech = ToStringField(GrammarListCode, secondaryPos); // It's fine if secondaryPos is still null here + lfSense.PartOfSpeech = ToOptionListItem(GrammarListCode, pos); + lfSense.SecondaryPartOfSpeech = ToOptionListItem(GrammarListCode, secondaryPos); // It's fine if secondaryPos is still null here } } - lfSense.PhonologyNote = ToMultiText(lcmSense.PhonologyNote, lcmSense.Guid); + lfSense.PhonologyNote = ToMultiText(lcmSense.PhonologyNote); if (lcmSense.PicturesOS != null) { lfSense.Pictures = new List(lcmSense.PicturesOS.Select(LcmPictureToLfPicture)); @@ -418,26 +428,26 @@ private LfSense LcmSenseToLfSense(ILexSense lcmSense) //foreach (var LcmPic in lcmSense.PicturesOS) // lfSense.Pictures.Add(LcmPictureToLfPicture(LcmPic)); } - lfSense.SenseBibliography = ToMultiText(lcmSense.Bibliography, lcmSense.Guid); - lfSense.SenseRestrictions = ToMultiText(lcmSense.Restrictions, lcmSense.Guid); + lfSense.SenseBibliography = ToMultiText(lcmSense.Bibliography); + lfSense.SenseRestrictions = ToMultiText(lcmSense.Restrictions); if (lcmSense.ReferringReversalIndexEntries != null) { - var reversalEntries = lcmSense.ReferringReversalIndexEntries.Select(e => LfStringField.CreateFrom(e.LongName, lcmSense.Guid)); + var reversalEntries = lcmSense.ReferringReversalIndexEntries.Select(e => LfStringField.CreateFrom(e.LongName)); lfSense.ReversalEntries = LfStringArrayField.CreateFrom(reversalEntries); } - lfSense.ScientificName = LfMultiText.FromSingleITsString(lcmSense.ScientificName, lcmSense.Guid, ServiceLocator.WritingSystemFactory); - lfSense.SemanticDomain = ToStringArrayField(SemDomListCode, lcmSense.SemanticDomainsRC); - lfSense.SemanticsNote = ToMultiText(lcmSense.SemanticsNote, lcmSense.Guid); + lfSense.ScientificName = LfMultiText.FromSingleITsString(lcmSense.ScientificName, ServiceLocator.WritingSystemFactory); + lfSense.SemanticDomain = ToOptionListItems(SemDomListCode, lcmSense.SemanticDomainsRC); + lfSense.SemanticsNote = ToMultiText(lcmSense.SemanticsNote); // lcmSense.SensesOS; // Not mapped because LF doesn't handle subsenses. TODO: When LF handles subsenses, map this one. - lfSense.SenseType = ToStringField(SenseTypeListCode, lcmSense.SenseTypeRA); - lfSense.SociolinguisticsNote = ToMultiText(lcmSense.SocioLinguisticsNote, lcmSense.Guid); + lfSense.SenseType = ToOptionListItem(SenseTypeListCode, lcmSense.SenseTypeRA); + lfSense.SociolinguisticsNote = ToMultiText(lcmSense.SocioLinguisticsNote); if (lcmSense.Source != null) { - lfSense.Source = LfMultiText.FromSingleITsString(lcmSense.Source, lcmSense.Guid, ServiceLocator.WritingSystemFactory); + lfSense.Source = LfMultiText.FromSingleITsString(lcmSense.Source, ServiceLocator.WritingSystemFactory); } - lfSense.Status = ToStringArrayField(StatusListCode, lcmSense.StatusRA); - lfSense.Usages = ToStringArrayField(UsageTypeListCode, lcmSense.UsageTypesRC); + lfSense.Status = ToOptionListItem(StatusListCode, lcmSense.StatusRA); + lfSense.Usages = ToOptionListItems(UsageTypeListCode, lcmSense.UsageTypesRC); /* Fields not mapped because it doesn't make sense to map them (e.g., Hvo, backreferences, etc): @@ -522,8 +532,8 @@ private LfExample LcmExampleToLfExample(ILexExampleSentence LcmExample) ILgWritingSystem VernacularWritingSystem = ServiceLocator.LanguageProject.DefaultVernacularWritingSystem; lfExample.Guid = LcmExample.Guid; - lfExample.Sentence = ToMultiText(LcmExample.Example, LcmExample.Guid); - lfExample.Reference = LfMultiText.FromSingleITsString(LcmExample.Reference, LcmExample.Guid, ServiceLocator.WritingSystemFactory); + lfExample.Sentence = ToMultiText(LcmExample.Example); + lfExample.Reference = LfMultiText.FromSingleITsString(LcmExample.Reference, ServiceLocator.WritingSystemFactory); // ILexExampleSentence fields we currently do not convert: // LcmExample.DoNotPublishInRC; // LcmExample.LiftResidue; @@ -537,7 +547,7 @@ private LfExample LcmExampleToLfExample(ILexExampleSentence LcmExample) // TODO: Once LF improves its data model for translations, persist all of them instead of just the first. foreach (ICmTranslation translation in LcmExample.TranslationsOC.Take(1)) { - lfExample.Translation = ToMultiText(translation.Translation, LcmExample.Guid); + lfExample.Translation = ToMultiText(translation.Translation); lfExample.TranslationGuid = translation.Guid; } @@ -553,7 +563,7 @@ private LfExample LcmExampleToLfExample(ILexExampleSentence LcmExample) private LfPicture LcmPictureToLfPicture(ICmPicture LcmPicture) { var result = new LfPicture(); - result.Caption = ToMultiText(LcmPicture.Caption, LcmPicture.Guid); + result.Caption = ToMultiText(LcmPicture.Caption); if ((LcmPicture.PictureFileRA != null) && (!string.IsNullOrEmpty(LcmPicture.PictureFileRA.InternalPath))) { result.FileName = LcmPictureFilenameToLfPictureFilename(LcmPicture.PictureFileRA.InternalPath); diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs index 1ae0de38..9a32a2d6 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmLexicon.cs @@ -716,10 +716,10 @@ private void LfSenseToLcmSense(LfSense lfSense, ILexEntry owner) LcmSense.ImportResidue = BestTsStringFromMultiText(lfSense.SenseImportResidue); SetMultiStringFrom(LcmSense.Restrictions, lfSense.SenseRestrictions); - LcmSense.SenseTypeRA = ListConverters[SenseTypeListCode].FromStringField(lfSense.SenseType); + LcmSense.SenseTypeRA = ListConverters[SenseTypeListCode].LookupByItem(lfSense.SenseType); SetMultiStringFrom(LcmSense.SocioLinguisticsNote, lfSense.SociolinguisticsNote); LcmSense.Source = BestTsStringFromMultiText(lfSense.Source); - LcmSense.StatusRA = ListConverters[StatusListCode].FromStringArrayFieldWithOneCase(lfSense.Status); + LcmSense.StatusRA = ListConverters[StatusListCode].LookupByItem(lfSense.Status); ListConverters[UsageTypeListCode].UpdatePossibilitiesFromStringArray(LcmSense.UsageTypesRC, lfSense.Usages); @@ -915,15 +915,15 @@ private void SetPronunciation(ILexEntry LcmEntry, LfLexEntry lfEntry) LcmPronunciation.Tone = BestTsStringFromMultiText(lfEntry.Tone); SetMultiStringFrom(LcmPronunciation.Form, lfEntry.Pronunciation); LcmPronunciation.LocationRA = - (ICmLocation)ListConverters[LocationListCode].FromStringField(lfEntry.Location); + (ICmLocation)ListConverters[LocationListCode].LookupByItem(lfEntry.Location); // Not handling LcmPronunciation.MediaFilesOS. TODO: At some point we may want to handle // media files as well. // Not handling LcmPronunciation.LiftResidue } - private IPartOfSpeech ConvertPos(LfStringField source, LfSense owner) + private IPartOfSpeech ConvertPos(LfOptionListItem source, LfSense owner) { - return ListConverters[GrammarListCode].FromStringField(source) as IPartOfSpeech; + return ListConverters[GrammarListCode].LookupByItem(source) as IPartOfSpeech; } } } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs index 63ed0204..85fae751 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmOptionList.cs @@ -24,8 +24,7 @@ public class ConvertMongoToLcmOptionList protected ICmPossibilityList _parentList; #endif - //todo - public Dictionary Possibilities { get; protected set; } + public Dictionary Possibilities { get; protected set; } #if false // Once we allow LanguageForge to create optionlist items with "canonical" values (parts of speech, semantic domains, etc.), uncomment this version of the constructor public ConvertMongoToLcmOptionList(IRepository possRepo, LfOptionList lfOptionList, ILogger logger, ICmPossibilityList parentList, int wsForKeys, CanonicalOptionListSource canonicalSource = null) @@ -47,14 +46,14 @@ public virtual void RebuildLookupTables(LfOptionList lfOptionList) { _lfOptionList = lfOptionList; - Possibilities = new Dictionary(); + Possibilities = new Dictionary(); if (lfOptionList == null || lfOptionList.Items == null) return; foreach (LfOptionListItem item in lfOptionList.Items) { ICmPossibility poss = LookupByItem(item); if (poss != null) - Possibilities[item.Guid] = poss; + Possibilities[item.Guid.Value] = poss; } } @@ -74,44 +73,19 @@ public ICmPossibility CreateFromCanonicalItem(CanonicalItem item) } #endif - public ICmPossibility FromStringField(LfStringField keyField) - { - if (keyField == null) - return null; - if (Possibilities.TryGetValue(keyField.LcmGuid, out var result)) - return result; - if (_canonicalSource != null) - return LookupByCanonicalItem(_canonicalSource.ByKeyOrNull(keyField.Value)); - - return null; - } - - public ICmPossibility FromStringArrayFieldWithOneCase(LfStringArrayField keyField) - { - if (keyField == null || keyField.Values == null || keyField.IsEmpty) - return null; - return Possibilities[keyField.LcmGuids.First()]; - } - - // Used in UpdatePossibilitiesFromStringArray and UpdateInvertedPossibilitiesFromStringArray below - // Generic so that they can handle lists like AnthroCodes, etc. - public IEnumerable FromStringArrayField(LfStringArrayField source) - { - if (source == null) return new List(); - - var result = source.LcmGuids.Select(g => (T) Possibilities[g]); - return result; - } - - protected ICmPossibility LookupByItem(LfOptionListItem item) + public ICmPossibility LookupByItem(LfOptionListItem item) { if (item == null) return null; ICmPossibility result; if (item.Guid.HasValue) { - if (_possRepo.TryGetObject(item.Guid.Value, out result)) + if (Possibilities.TryGetValue(item.Guid.Value, out result)) + return result; + else if (_possRepo.TryGetObject(item.Guid.Value, out result)) return result; + else if (_canonicalSource != null) + return LookupByCanonicalItem(_canonicalSource.ByKeyOrNull(item.Value)); } #if false // Once we are populating Lcm from LF, we might also need to fall back to abbreviation and name for these lookups, because Guids might not be available return FromAbbrevAndName(item.Abbreviation, item.Value); @@ -143,7 +117,7 @@ protected ICmPossibility LookupByCanonicalItem(CanonicalItem item) // from ICmPossibility (e.g., ICmAnthroItem). This results in type errors at compile time: parameter // types like ILcmReferenceCollection don't match ILcmReferenceCollection. // Generics solve the problem, and can be automatically inferred by the compiler to boot. - public void SetPossibilitiesCollection(ILcmReferenceCollection dest, IEnumerable newItems) + private void SetPossibilitiesCollection(ILcmReferenceCollection dest, IEnumerable newItems) where T: class, ICmPossibility { // If we know of NO valid possibility keys, don't make any changes. That's because knowing of NO valid possibility keys @@ -164,10 +138,11 @@ public void SetPossibilitiesCollection(ILcmReferenceCollection dest, IEnum } // Assumption: "source" contains valid keys. CAUTION: No error checking is done to ensure that this is true. - public void UpdatePossibilitiesFromStringArray(ILcmReferenceCollection dest, LfStringArrayField source) + public void UpdatePossibilitiesFromStringArray(ILcmReferenceCollection dest, List source) where T: class, ICmPossibility { - SetPossibilitiesCollection(dest, FromStringArrayField(source)); + var list = from s in source select (T)LookupByItem(s); + SetPossibilitiesCollection(dest, list); } // Once we allow LanguageForge to create optionlist items with "canonical" values (parts of speech, semantic domains, etc.), uncomment this block diff --git a/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs b/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs index b327586b..36fbf5d2 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfFieldBase.cs @@ -28,6 +28,11 @@ protected bool _ShouldSerializeLfStringField(LfStringField value) return value != null && !value.IsEmpty; } + protected bool _ShouldSerialize(LfOptionListItem value) + { + return value != null && !value.IsEmpty; + } + protected bool _ShouldSerializeList(IEnumerable value) { return value != null && value.GetEnumerator().MoveNext() != false; diff --git a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs index 41aa85b0..29411c8e 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfLexEntry.cs @@ -36,7 +36,7 @@ public class LfLexEntry : LfFieldBase, IHasNullableGuid public LfMultiText EtymologyComment { get; set; } public LfMultiText EtymologySource { get; set; } public LfMultiText LiteralMeaning { get; set; } - public LfStringField Location { get; set; } + public LfOptionListItem Location { get; set; } public string MorphologyType { get; set; } public LfMultiText Note { get; set; } public LfMultiText Pronunciation { get; set; } @@ -67,7 +67,7 @@ public LfLexEntry() public bool ShouldSerializeEtymologyComment() { return _ShouldSerializeLfMultiText(EtymologyComment); } public bool ShouldSerializeEtymologySource() { return _ShouldSerializeLfMultiText(EtymologySource); } public bool ShouldSerializeLiteralMeaning() { return _ShouldSerializeLfMultiText(LiteralMeaning); } - public bool ShouldSerializeLocation() { return _ShouldSerializeLfStringField(Location); } + public bool ShouldSerializeLocation() { return _ShouldSerialize(Location); } public bool ShouldSerializeMorphologyType() { return !String.IsNullOrEmpty(MorphologyType); } public bool ShouldSerializeNote() { return _ShouldSerializeLfMultiText(Note); } public bool ShouldSerializePronunciation() { return _ShouldSerializeLfMultiText(Pronunciation); } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs index 287d0e44..f2cedf79 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfMultiText.cs @@ -14,7 +14,7 @@ public class LfMultiText : Dictionary // Note: NOT derive { public bool IsEmpty { get { return (Count <= 0) || (this.All(kv => kv.Value == null || kv.Value.IsEmpty)); } } - public static LfMultiText FromLcmMultiString(IMultiAccessorBase other, Guid lcmId, ILgWritingSystemFactory wsManager) + public static LfMultiText FromLcmMultiString(IMultiAccessorBase other, ILgWritingSystemFactory wsManager) { LfMultiText newInstance = new LfMultiText(); foreach (int wsid in other.AvailableWritingSystemIds) @@ -22,34 +22,34 @@ public static LfMultiText FromLcmMultiString(IMultiAccessorBase other, Guid lcmI string wsstr = wsManager.GetStrFromWs(wsid); ITsString value = other.get_String(wsid); string text = LfMerge.Core.DataConverters.ConvertLcmToMongoTsStrings.TextFromTsString(value, wsManager); - LfStringField field = LfStringField.CreateFrom(text, lcmId); + LfStringField field = LfStringField.CreateFrom(text); if (field != null) newInstance.Add(wsstr, field); } return newInstance; } - public static LfMultiText FromSingleStringMapping(string key, string value, Guid lcmId) + public static LfMultiText FromSingleStringMapping(string key, string value) { - LfStringField field = LfStringField.CreateFrom(value, lcmId); + LfStringField field = LfStringField.CreateFrom(value); if (field == null) return null; return new LfMultiText { { key, field } }; } - public static LfMultiText FromSingleITsString(ITsString value, Guid lcmId, ILgWritingSystemFactory wsManager) + public static LfMultiText FromSingleITsString(ITsString value, ILgWritingSystemFactory wsManager) { if (value == null || value.Text == null) return null; int wsId = value.get_WritingSystem(0); string wsStr = wsManager.GetStrFromWs(wsId); string text = LfMerge.Core.DataConverters.ConvertLcmToMongoTsStrings.TextFromTsString(value, wsManager); - LfStringField field = LfStringField.CreateFrom(text, lcmId); + LfStringField field = LfStringField.CreateFrom(text); if (field == null) return null; return new LfMultiText { { wsStr, field } }; } - public static LfMultiText FromMultiITsString(ITsMultiString value, Guid lcmId, ILgWritingSystemFactory wsManager) + public static LfMultiText FromMultiITsString(ITsMultiString value, ILgWritingSystemFactory wsManager) { if (value == null || value.StringCount == 0) return null; LfMultiText mt = new LfMultiText(); @@ -61,7 +61,7 @@ public static LfMultiText FromMultiITsString(ITsMultiString value, Guid lcmId, I if (!string.IsNullOrEmpty(wsStr)) { string valueStr = LfMerge.Core.DataConverters.ConvertLcmToMongoTsStrings.TextFromTsString(tss, wsManager); - LfStringField field = LfStringField.CreateFrom(valueStr, lcmId); + LfStringField field = LfStringField.CreateFrom(valueStr); if (field != null) mt.Add(wsStr, field); } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs b/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs index 429550db..6ab66600 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfOptionListItem.cs @@ -14,7 +14,10 @@ public class LfOptionListItem : IHasNullableGuid public Guid? Guid { get; set; } public string Key { - get => key; + get + { + return key ?? Guid.Value.ToString(); + } set { if (System.Guid.TryParse(value, out var guid)) @@ -33,6 +36,8 @@ public string Key } public string Value { get; set; } public string Abbreviation { get; set; } + + public bool IsEmpty { get { return String.IsNullOrEmpty(Value); } } } } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfSense.cs b/src/LfMerge.Core/LanguageForge/Model/LfSense.cs index ba9ed72b..91b6d0c0 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfSense.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfSense.cs @@ -17,11 +17,11 @@ public class LfSense : LfFieldBase, IHasNullableGuid public Guid? Guid { get; set; } // Data properties - public LfStringField PartOfSpeech { get; set; } - public LfStringField SecondaryPartOfSpeech { get; set; } + public LfOptionListItem PartOfSpeech { get; set; } + public LfOptionListItem SecondaryPartOfSpeech { get; set; } [BsonRepresentation(BsonType.String)] public Guid? PartOfSpeechGuid { get; set; } // Present for historical reasons, but never persisted - public LfStringArrayField SemanticDomain { get; set; } + public List SemanticDomain { get; set; } public List Examples { get; set; } public BsonDocument CustomFields { get; set; } // Mapped at runtime public BsonDocument CustomFieldGuids { get; set; } @@ -42,20 +42,20 @@ public class LfSense : LfFieldBase, IHasNullableGuid public LfMultiText SociolinguisticsNote { get; set; } public LfMultiText Source { get; set; } public LfMultiText SenseImportResidue { get; set; } - public LfStringArrayField Usages { get; set; } + public List Usages { get; set; } public LfStringArrayField ReversalEntries { get; set; } - public LfStringField SenseType { get; set; } - public LfStringArrayField AcademicDomains { get; set; } + public LfOptionListItem SenseType { get; set; } + public List AcademicDomains { get; set; } public LfStringArrayField SensePublishIn { get; set; } - public LfStringArrayField AnthropologyCategories { get; set; } - public LfStringArrayField Status { get; set; } + public List AnthropologyCategories { get; set; } + public LfOptionListItem Status { get; set; } // Ugh. But Mongo doesn't let you provide a ShouldSerialize() by field *type*, only by field *name*. // Maybe later we can write reflection code to automatically add these to the class... - public bool ShouldSerializePartOfSpeech() { return _ShouldSerializeLfStringField(PartOfSpeech); } - public bool ShouldSerializeSecondaryPartOfSpeech() { return _ShouldSerializeLfStringField(SecondaryPartOfSpeech); } + public bool ShouldSerializePartOfSpeech() { return _ShouldSerialize(PartOfSpeech); } + public bool ShouldSerializeSecondaryPartOfSpeech() { return _ShouldSerialize(SecondaryPartOfSpeech); } public bool ShouldSerializePartOfSpeechGuid() { return false; } - public bool ShouldSerializeSemanticDomain() { return _ShouldSerializeLfStringArrayField(SemanticDomain); } + public bool ShouldSerializeSemanticDomain() { return _ShouldSerializeList(SemanticDomain); } public bool ShouldSerializeExamples() { return _ShouldSerializeList(Examples); } public bool ShouldSerializeCustomFields() { return _ShouldSerializeBsonDocument(CustomFields); } public bool ShouldSerializeCustomFieldGuids() { return _ShouldSerializeBsonDocument(CustomFieldGuids); } @@ -76,13 +76,13 @@ public class LfSense : LfFieldBase, IHasNullableGuid public bool ShouldSerializeSociolinguisticsNote() { return _ShouldSerializeLfMultiText(SociolinguisticsNote); } public bool ShouldSerializeSource() { return _ShouldSerializeLfMultiText(Source); } public bool ShouldSerializeSenseImportResidue() { return _ShouldSerializeLfMultiText(SenseImportResidue); } - public bool ShouldSerializeUsages() { return _ShouldSerializeLfStringArrayField(Usages); } + public bool ShouldSerializeUsages() { return _ShouldSerializeList(Usages); } public bool ShouldSerializeReversalEntries() { return _ShouldSerializeLfStringArrayField(ReversalEntries); } - public bool ShouldSerializeSenseType() { return _ShouldSerializeLfStringField(SenseType); } - public bool ShouldSerializeAcademicDomains() { return _ShouldSerializeLfStringArrayField(AcademicDomains); } + public bool ShouldSerializeSenseType() { return _ShouldSerialize(SenseType); } + public bool ShouldSerializeAcademicDomains() { return _ShouldSerializeList(AcademicDomains); } public bool ShouldSerializeSensePublishIn() { return false; } // Get rid of this one if we find it - public bool ShouldSerializeAnthropologyCategories() { return _ShouldSerializeLfStringArrayField(AnthropologyCategories); } - public bool ShouldSerializeStatus() { return _ShouldSerializeLfStringArrayField(Status); } + public bool ShouldSerializeAnthropologyCategories() { return _ShouldSerializeList(AnthropologyCategories); } + public bool ShouldSerializeStatus() { return _ShouldSerialize(Status); } public LfSense() { diff --git a/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs b/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs index 4e901293..113b0892 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfStringArrayField.cs @@ -11,7 +11,6 @@ public class LfStringArrayField : LfFieldBase { private IList _values = new List(); - public IEnumerable LcmGuids { get { return _values.Select(v => v.LcmGuid); } } public List Values { get; set; } public bool IsEmpty { get { return Values.Count <= 0; } } diff --git a/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs b/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs index baddcffd..3b4417b2 100644 --- a/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs +++ b/src/LfMerge.Core/LanguageForge/Model/LfStringField.cs @@ -9,8 +9,6 @@ public class LfStringField : LfFieldBase { public string Value { get; private set; } - public Guid LcmGuid { get; private set; } - public bool IsEmpty { get { return String.IsNullOrEmpty(Value); } } public override string ToString() @@ -18,11 +16,11 @@ public override string ToString() return Value; } - public static LfStringField CreateFrom(string source, Guid lcmId) + public static LfStringField CreateFrom(string source) { - if (source == null || lcmId == null) + if (source == null) return null; - return new LfStringField { Value = source, LcmGuid = lcmId }; + return new LfStringField { Value = source }; } private LfStringField() { } From 3ce8afaa3b35d2f9d886b7f6a94d13114c54c772 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Wed, 16 Nov 2022 09:59:51 -0600 Subject: [PATCH 5/6] Adding consideration for GUID values in custom lists. I can't test this, since the List deserialization just decided to stop working, but I think this is a good general concept. Assuming LF can't add list items on its own, any time a GUID is stored in MongoDB, it's from LCM. So if the value from MongoDB is a GUID, find the matching value from the LCM list. --- .../ConvertMongoToLcmCustomField.cs | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs index a8de39d0..004e400c 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmCustomField.cs @@ -215,7 +215,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui fieldWs = WritingSystemServices.ActualWs(cache, fieldWs, hvo, flid); } ICmPossibilityList parentList = GetParentListForField(flid); - ICmPossibility newPoss = parentList.FindOrCreatePossibility(nameHierarchy, fieldWs); + ICmPossibility newPoss = FindPossibility(value.AsString, parentList, fieldWs); int oldHvo = data.get_ObjectProp(hvo, flid); if (newPoss.Hvo == oldHvo) @@ -308,7 +308,7 @@ public bool SetCustomFieldData(int hvo, int flid, BsonValue value, BsonValue gui // So we assume they exist in FW, and just look them up. foreach (string key in keysFromLF) { - ICmPossibility poss = parentList.FindOrCreatePossibility(key, wsEn); + ICmPossibility poss = FindPossibility(key, parentList, wsEn); // TODO: If this is a new possibility, then we need to populate it with ALL the corresponding data from LF, // which we don't necessarily have at this point. Need to make that a separate step in the Send/Receive: converting option lists first. fieldObjs.Add(poss); @@ -420,5 +420,23 @@ public void SetCustomFieldsForThisCmObject(ICmObject cmObj, string objectType, B logger.Warning("Custom field {0} from LF skipped, because we're not yet creating new custom fields in LCM", fieldName); } } + + private ICmPossibility FindPossibility(string key, ICmPossibilityList parentList, int fieldWs) + { + ICmPossibility newPoss; + var guid = ParseGuidOrDefault(key); + if (guid != default(Guid)) + { + // assuming LF doesn't have the ability to add list options, + // a GUID in the DB should have a match in LCM + newPoss = parentList.ReallyReallyAllPossibilities.First(p => p.Guid.Equals(guid)); + } + else + { + newPoss = parentList.FindOrCreatePossibility(key, fieldWs); + } + + return newPoss; + } } } \ No newline at end of file From 23a26f179d7992921ce24e393ab23e3cea7be465 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Thu, 17 Nov 2022 12:45:40 -0600 Subject: [PATCH 6/6] LcmToMongo action now supports incomplete DB's. This fixes an exception thrown when trying to deserialize OptionListItem's with partial DB entries. For initial clones, the DB is somewhat filled out (why?), so this adds a try/catch to allow the DB request to fail, in which case an empty LF list will be used. Also had to change slightly the UpdateRecord call of MongoConnection from using FindOneAndUpdate to UpdateOne. It appeared the only difference was the return type, which was unused, and the former was deserializing its target type before serializing it, causing the same DB exception. --- .../DataConverters/ConvertLcmToMongoLexicon.cs | 18 +++++++++++++++--- .../MongoConnector/MongoConnection.cs | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs index 89adae69..149cb18b 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoLexicon.cs @@ -624,11 +624,23 @@ private Dictionary LcmWsToLfWs() return lfWsList; } - private ConvertLcmToMongoOptionList ConvertOptionListFromLcm(ILfProject project, string listCode, ICmPossibilityList LcmOptionList, bool updateMongoList = true) + private ConvertLcmToMongoOptionList ConvertOptionListFromLcm(ILfProject project, string listCode, ICmPossibilityList lcmOptionList, bool updateMongoList = true) { - LfOptionList lfExistingOptionList = Connection.GetLfOptionListByCode(project, listCode); + LfOptionList lfExistingOptionList = null; + try + { + lfExistingOptionList = Connection.GetLfOptionListByCode(project, listCode); //doesn't work unless the DB is fully populated + } + catch (Exception) { } + var converter = new ConvertLcmToMongoOptionList(lfExistingOptionList, _wsEn, listCode, Logger, ServiceLocator.WritingSystemFactory); - LfOptionList lfChangedOptionList = converter.PrepareOptionListUpdate(LcmOptionList); + LfOptionList lfChangedOptionList = converter.PrepareOptionListUpdate(lcmOptionList); + + if (lfExistingOptionList == null) + { + lfChangedOptionList.DateModified = lcmOptionList.DateModified; //if DB didn't have an entry, preserve LCM date + } + if (updateMongoList) Connection.UpdateRecord(project, lfChangedOptionList, listCode); return new ConvertLcmToMongoOptionList(lfChangedOptionList, _wsEn, listCode, Logger, ServiceLocator.WritingSystemFactory); diff --git a/src/LfMerge.Core/MongoConnector/MongoConnection.cs b/src/LfMerge.Core/MongoConnector/MongoConnection.cs index f9616bca..b8d88754 100644 --- a/src/LfMerge.Core/MongoConnector/MongoConnection.cs +++ b/src/LfMerge.Core/MongoConnector/MongoConnection.cs @@ -970,10 +970,10 @@ private bool UpdateRecordImpl(ILfProject project, TDocument data, Fil mongoDb = GetMainDatabase(); UpdateDefinition update = BuildUpdate(data, false); IMongoCollection collection = mongoDb.GetCollection(collectionName); - var updateOptions = new FindOneAndUpdateOptions { + var updateOptions = new UpdateOptions { IsUpsert = true }; - collection.FindOneAndUpdate(filter, update, updateOptions); + collection.UpdateOne(filter, update, updateOptions); return true; }