Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
@microsoft/bf-orchestrator
Advanced tools
@microsoft/bf-orchestrator - npm Package Compare versions
Comparing version 4.11.0-beta.20201025.035cdcb to 4.11.0-beta.20201026.d8a7aef
@@ -6,3 +6,3 @@ export declare class OrchestratorAssess { | ||
| static readonly assessmentSetEntityLabelsOutputFilename: string; | ||
| static runAsync(inputPathConfiguration: string, predictionPathConfiguration: string, outputPath: string): Promise<void>; | ||
| static runAsync(inputPathConfiguration: string, predictionPathConfiguration: string, outputPath: string, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| } |
@@ -11,2 +11,3 @@ "use strict"; | ||
| const orchestratorhelper_1 = require("./orchestratorhelper"); | ||
| const utilitylabelresolver_1 = require("./utilitylabelresolver"); | ||
| const utility_1 = require("./utility"); | ||
@@ -16,3 +17,3 @@ class OrchestratorAssess { | ||
| // eslint-disable-next-line max-params | ||
| static async runAsync(inputPathConfiguration, predictionPathConfiguration, outputPath) { | ||
| static async runAsync(inputPathConfiguration, predictionPathConfiguration, outputPath, obfuscateEvaluationReport = false) { | ||
| // ----------------------------------------------------------------------- | ||
@@ -32,2 +33,4 @@ // ---- NOTE ---- process arguments -------------------------------------- | ||
| utility_1.Utility.debuggingLog(`outputPath=${outputPath}`); | ||
| utility_1.Utility.toObfuscateLabelTextInReportUtility = obfuscateEvaluationReport; | ||
| utilitylabelresolver_1.UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver = obfuscateEvaluationReport; | ||
| // ---- NOTE ---- load the ground truth set ------------------------------ | ||
@@ -34,0 +37,0 @@ const groundTruthFileConfiguration = inputPathConfiguration; |
@@ -7,3 +7,3 @@ export declare class OrchestratorEvaluate { | ||
| static readonly snapshotSetLabelsOutputFilename: string; | ||
| static runAsync(inputPath: string, outputPath: string, baseModelPath?: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbeddings?: boolean): Promise<void>; | ||
| static runAsync(inputPath: string, outputPath: string, baseModelPath?: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbeddings?: boolean, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| } |
@@ -17,3 +17,3 @@ "use strict"; | ||
| // eslint-disable-next-line max-params | ||
| static async runAsync(inputPath, outputPath, baseModelPath = '', ambiguousClosenessThresholdParameter = utility_1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = utility_1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = utility_1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = utility_1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbeddings = false) { | ||
| static async runAsync(inputPath, outputPath, baseModelPath = '', ambiguousClosenessThresholdParameter = utility_1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = utility_1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = utility_1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = utility_1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbeddings = false, obfuscateEvaluationReport = false) { | ||
| // ----------------------------------------------------------------------- | ||
@@ -45,2 +45,5 @@ // ---- NOTE ---- process arguments | ||
| utility_1.Utility.debuggingLog(`fullEmbeddings=${fullEmbeddings}`); | ||
| utility_1.Utility.debuggingLog(`obfuscateEvaluationReport=${obfuscateEvaluationReport}`); | ||
| utility_1.Utility.toObfuscateLabelTextInReportUtility = obfuscateEvaluationReport; | ||
| utilitylabelresolver_1.UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver = obfuscateEvaluationReport; | ||
| // ----------------------------------------------------------------------- | ||
@@ -47,0 +50,0 @@ // ---- NOTE ---- load the snapshot set |
@@ -12,3 +12,9 @@ import { LabelType } from './labeltype'; | ||
| constructor(labeltype: LabelType, name: string, span: Span); | ||
| toOutputString(toObfuscate?: boolean): string; | ||
| toSimpleString(): string; | ||
| toObfuscatedString(): string; | ||
| static outputNumber(input: number, toObfuscate?: boolean): number; | ||
| static outputString(input: string, toObfuscate?: boolean): string; | ||
| static simpleString(input: string): string; | ||
| static obfuscatedString(input: string): string; | ||
| toObject(): { | ||
@@ -15,0 +21,0 @@ 'name': string; |
@@ -10,2 +10,4 @@ "use strict"; | ||
| const span_1 = require("./span"); | ||
| const bf_dispatcher_1 = require("@microsoft/bf-dispatcher"); | ||
| // import {Utility as UtilityDispatcher} from '@microsoft/bf-dispatcher'; | ||
| class Label { | ||
@@ -29,5 +31,36 @@ constructor(labeltype, name, span) { | ||
| } | ||
| toOutputString(toObfuscate = false) { | ||
| if (toObfuscate) { | ||
| return this.toObfuscatedString(); | ||
| } | ||
| return this.toSimpleString(); | ||
| } | ||
| toSimpleString() { | ||
| return `${this.name}-${this.labeltype}-${this.span.offset}-${this.span.length}`; | ||
| } | ||
| toObfuscatedString() { | ||
| const nameObfuscated = bf_dispatcher_1.CryptoUtility.getStringObfuscated(this.name); | ||
| const offsetObfuscated = bf_dispatcher_1.CryptoUtility.getNumberObfuscated(this.span.offset); | ||
| const lengthObfuscated = bf_dispatcher_1.CryptoUtility.getNumberObfuscated(this.span.length); | ||
| return `${nameObfuscated}-${this.labeltype}-${offsetObfuscated}-${lengthObfuscated}`; | ||
| } | ||
| static outputNumber(input, toObfuscate = false) { | ||
| if (toObfuscate) { | ||
| return bf_dispatcher_1.CryptoUtility.getNumberObfuscated(input); | ||
| } | ||
| return input; | ||
| } | ||
| static outputString(input, toObfuscate = false) { | ||
| if (toObfuscate) { | ||
| return Label.obfuscatedString(input); | ||
| } | ||
| return input; | ||
| } | ||
| static simpleString(input) { | ||
| return input; | ||
| } | ||
| static obfuscatedString(input) { | ||
| const inputObfuscated = bf_dispatcher_1.CryptoUtility.getStringObfuscated(input); | ||
| return inputObfuscated; | ||
| } | ||
| toObject() { | ||
@@ -34,0 +67,0 @@ return { |
| export declare class Orchestrator { | ||
| static createAsync(baseModelPath: string, inputPathConfiguration: string, outputPath: string, hierarchical?: boolean, fullEmbedding?: boolean): Promise<void>; | ||
| static buildAsync(baseModelPath: string, inputs: any[], isDialog: boolean, luConfig?: any, fullEmbedding?: boolean): Promise<any>; | ||
| static evaluateAsync(inputPath: string, outputPath: string, baseModelPath?: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean): Promise<void>; | ||
| static evaluateAsync(inputPath: string, outputPath: string, baseModelPath?: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| static baseModelGetAsync(baseModelPath: string, nlrId: string, onProgress?: any, onFinish?: any): Promise<void>; | ||
| static baseModelListAsync(): Promise<string>; | ||
| static baseModelGetVersionsAsync(): Promise<any>; | ||
| static predictAsync(baseModelPath: string, inputPath: string, outputPath: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean): Promise<void>; | ||
| static testAsync(baseModelPath: string, inputPathConfiguration: string, testPathConfiguration: string, outputPath: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean): Promise<void>; | ||
| static predictAsync(baseModelPath: string, inputPath: string, outputPath: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| static testAsync(baseModelPath: string, inputPathConfiguration: string, testPathConfiguration: string, outputPath: string, ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| static queryAsync(baseModelPath: string, inputPathConfiguration: string, queryConfiguration: string, // outputPath: string, | ||
| ambiguousClosenessThresholdParameter?: number, lowConfidenceScoreThresholdParameter?: number, multiLabelPredictionThresholdParameter?: number, unknownLabelPredictionThresholdParameter?: number, fullEmbedding?: boolean): Promise<void>; | ||
| static assessAsync(inputPathConfiguration: string, predictionPathConfiguration: string, outputPath: string): Promise<void>; | ||
| static assessAsync(inputPathConfiguration: string, predictionPathConfiguration: string, outputPath: string, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| } |
@@ -27,4 +27,4 @@ "use strict"; | ||
| // eslint-disable-next-line max-params | ||
| static async evaluateAsync(inputPath, outputPath, baseModelPath = '', ambiguousClosenessThresholdParameter = _1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = _1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = _1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = _1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbedding = false) { | ||
| await evaluate_1.OrchestratorEvaluate.runAsync(inputPath, outputPath, baseModelPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbedding); | ||
| static async evaluateAsync(inputPath, outputPath, baseModelPath = '', ambiguousClosenessThresholdParameter = _1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = _1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = _1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = _1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbedding = false, obfuscateEvaluationReport = false) { | ||
| await evaluate_1.OrchestratorEvaluate.runAsync(inputPath, outputPath, baseModelPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbedding, obfuscateEvaluationReport); | ||
| } | ||
@@ -46,8 +46,8 @@ /* | ||
| // eslint-disable-next-line max-params | ||
| static async predictAsync(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter = _1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = _1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = _1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = _1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbedding = false) { | ||
| await predict_1.OrchestratorPredict.runAsync(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbedding); | ||
| static async predictAsync(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter = _1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = _1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = _1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = _1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbedding = false, obfuscateEvaluationReport = false) { | ||
| await predict_1.OrchestratorPredict.runAsync(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbedding, obfuscateEvaluationReport); | ||
| } | ||
| // eslint-disable-next-line max-params | ||
| static async testAsync(baseModelPath, inputPathConfiguration, testPathConfiguration, outputPath, ambiguousClosenessThresholdParameter = _1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = _1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = _1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = _1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbedding = false) { | ||
| await test_1.OrchestratorTest.runAsync(baseModelPath, inputPathConfiguration, testPathConfiguration, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbedding); | ||
| static async testAsync(baseModelPath, inputPathConfiguration, testPathConfiguration, outputPath, ambiguousClosenessThresholdParameter = _1.Utility.DefaultAmbiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter = _1.Utility.DefaultLowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter = _1.Utility.DefaultMultiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter = _1.Utility.DefaultUnknownLabelPredictionThresholdParameter, fullEmbedding = false, obfuscateEvaluationReport = false) { | ||
| await test_1.OrchestratorTest.runAsync(baseModelPath, inputPathConfiguration, testPathConfiguration, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbedding, obfuscateEvaluationReport); | ||
| } | ||
@@ -61,4 +61,4 @@ // eslint-disable-next-line max-params | ||
| // eslint-disable-next-line max-params | ||
| static async assessAsync(inputPathConfiguration, predictionPathConfiguration, outputPath) { | ||
| await assess_1.OrchestratorAssess.runAsync(inputPathConfiguration, predictionPathConfiguration, outputPath); | ||
| static async assessAsync(inputPathConfiguration, predictionPathConfiguration, outputPath, obfuscateEvaluationReport = false) { | ||
| await assess_1.OrchestratorAssess.runAsync(inputPathConfiguration, predictionPathConfiguration, outputPath, obfuscateEvaluationReport); | ||
| } | ||
@@ -65,0 +65,0 @@ } |
@@ -17,3 +17,4 @@ import { IConfusionMatrix } from '@microsoft/bf-dispatcher'; | ||
| static readonly questionForMultiLabelPredictionThreshold: string; | ||
| static readonly questionForunknownLabelPredictionThreshold: string; | ||
| static readonly questionForUnknownLabelPredictionThreshold: string; | ||
| static readonly questionForObfuscateEvaluationReport: string; | ||
| protected inputPath: string; | ||
@@ -27,2 +28,3 @@ protected outputPath: string; | ||
| protected fullEmbeddings: boolean; | ||
| protected obfuscateEvaluationReport: boolean; | ||
| protected snapshotFile: string; | ||
@@ -99,3 +101,3 @@ protected predictingSetGroundTruthJsonContentOutputFilename: string; | ||
| }; | ||
| constructor(baseModelPath: string, inputPath: string, outputPath: string, ambiguousClosenessThresholdParameter: number, lowConfidenceScoreThresholdParameter: number, multiLabelPredictionThresholdParameter: number, unknownLabelPredictionThresholdParameter: number, fullEmbeddings?: boolean); | ||
| constructor(baseModelPath: string, inputPath: string, outputPath: string, ambiguousClosenessThresholdParameter: number, lowConfidenceScoreThresholdParameter: number, multiLabelPredictionThresholdParameter: number, unknownLabelPredictionThresholdParameter: number, fullEmbeddings?: boolean, obfuscateEvaluationReport?: boolean); | ||
| getPredictingSetGroundTruthJsonContentOutputFilename(): string; | ||
@@ -108,3 +110,3 @@ getPredictingSetPredictionJsonContentOutputFilename(): string; | ||
| buildLabelResolver(): Promise<void>; | ||
| static runAsync(baseModelPath: string, inputPath: string, outputPath: string, ambiguousClosenessThresholdParameter: number, lowConfidenceScoreThresholdParameter: number, multiLabelPredictionThresholdParameter: number, unknownLabelPredictionThresholdParameter: number, fullEmbeddings?: boolean): Promise<number>; | ||
| static runAsync(baseModelPath: string, inputPath: string, outputPath: string, ambiguousClosenessThresholdParameter: number, lowConfidenceScoreThresholdParameter: number, multiLabelPredictionThresholdParameter: number, unknownLabelPredictionThresholdParameter: number, fullEmbeddings?: boolean, obfuscateEvaluationReport?: boolean): Promise<number>; | ||
| commandLetLoop(): Promise<number>; | ||
@@ -143,2 +145,4 @@ commandLetH(): number; | ||
| commandLetVUTwithEntry(entry: string): number; | ||
| commandLetVO(question: any): Promise<number>; | ||
| commandLetVOwithEntry(entry: string): number; | ||
| commandLetA(): number; | ||
@@ -145,0 +149,0 @@ commandLetR(): number; |
@@ -18,4 +18,5 @@ "use strict"; | ||
| // import {Span} from './span'; | ||
| const bf_dispatcher_2 = require("@microsoft/bf-dispatcher"); | ||
| const utility_1 = require("./utility"); | ||
| const utilitylabelresolver_1 = require("./utilitylabelresolver"); | ||
| const utility_1 = require("./utility"); | ||
| /* eslint-disable no-console */ | ||
@@ -25,3 +26,3 @@ class OrchestratorPredict { | ||
| /* eslint-disable complexity */ | ||
| constructor(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings = false) { | ||
| constructor(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings = false, obfuscateEvaluationReport = false) { | ||
| this.inputPath = ''; | ||
@@ -35,2 +36,3 @@ this.outputPath = ''; | ||
| this.fullEmbeddings = false; | ||
| this.obfuscateEvaluationReport = false; | ||
| this.snapshotFile = ''; | ||
@@ -87,2 +89,3 @@ this.predictingSetGroundTruthJsonContentOutputFilename = ''; | ||
| utility_1.Utility.debuggingLog(`fullEmbeddings=${fullEmbeddings}`); | ||
| utility_1.Utility.debuggingLog(`obfuscateEvaluationReport=${obfuscateEvaluationReport}`); | ||
| this.inputPath = inputPath; | ||
@@ -96,2 +99,3 @@ this.outputPath = outputPath; | ||
| this.fullEmbeddings = fullEmbeddings; | ||
| this.obfuscateEvaluationReport = obfuscateEvaluationReport; | ||
| // ---- NOTE ---- load the snapshot set | ||
@@ -134,16 +138,16 @@ this.snapshotFile = this.inputPath; | ||
| // ---- NOTE ---- create a LabelResolver object. | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), ready to call LabelResolver.createAsync()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), ready to call LabelResolver.createAsync()'); | ||
| await labelresolver_1.LabelResolver.createAsync(this.baseModelPath); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), after calling LabelResolver.createAsync()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after calling LabelResolver.createAsync()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utilitylabelresolver_1.UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings(this.fullEmbeddings); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), ready to call OrchestratorHelper.getSnapshotFromFile()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), ready to call OrchestratorHelper.getSnapshotFromFile()'); | ||
| const snapshot = orchestratorhelper_1.OrchestratorHelper.getSnapshotFromFile(this.snapshotFile); | ||
| utility_1.Utility.debuggingLog(`LabelResolver.createWithSnapshotAsync(): typeof(snapshot)=${typeof snapshot}`); | ||
| utility_1.Utility.debuggingLog(`LabelResolver.createWithSnapshotAsync(): snapshot.byteLength=${snapshot.byteLength}`); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), after calling OrchestratorHelper.getSnapshotFromFile()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), ready to call LabelResolver.addSnapshot()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after calling OrchestratorHelper.getSnapshotFromFile()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), ready to call LabelResolver.addSnapshot()'); | ||
| await labelresolver_1.LabelResolver.addSnapshot(snapshot); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), after calling LabelResolver.addSnapshot()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after calling LabelResolver.addSnapshot()'); | ||
| } | ||
@@ -153,11 +157,11 @@ else { | ||
| await labelresolver_1.LabelResolver.createAsync(this.baseModelPath); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), after calling LabelResolver.createAsync()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after calling LabelResolver.createAsync()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), ready to call UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utilitylabelresolver_1.UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings(this.fullEmbeddings); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.runAsync(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after calling UtilityLabelResolver.resetLabelResolverSettingUseCompactEmbeddings()'); | ||
| } | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.buildLabelResolver(), after creating a LabelResolver object'); | ||
| } | ||
| static async runAsync(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings = false) { | ||
| const orchestratorPredict = new OrchestratorPredict(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings); | ||
| static async runAsync(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings = false, obfuscateEvaluationReport = false) { | ||
| const orchestratorPredict = new OrchestratorPredict(baseModelPath, inputPath, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings, obfuscateEvaluationReport); | ||
| // ---- NOTE ---- create a LabelResolver object. | ||
@@ -264,2 +268,6 @@ await orchestratorPredict.buildLabelResolver(); | ||
| break; | ||
| // eslint-disable-next-line no-await-in-loop | ||
| case 'vo': | ||
| await this.commandLetVO(question); | ||
| break; | ||
| case 'a': | ||
@@ -293,3 +301,3 @@ this.commandLetA(); | ||
| console.log(' Commandlets: h, q, d, s, u, cu, i, ci, ni, cni, q, p, v,'); | ||
| console.log(' vd, va, vm, vl, vat, vlt, vmt, vut, a, r, c, rl, n'); | ||
| console.log(' vd, va, vm, vl, vat, vlt, vmt, vut, vo, a, r, c, rl, n'); | ||
| console.log(' h - print this help message'); | ||
@@ -328,2 +336,4 @@ console.log(' q - quit'); | ||
| console.log(' vut - enter a new unknown-label threshold'); | ||
| console.log(' vo - enter a boolean for obfuscating labels/utterances or not in evaluation reports'); | ||
| console.log(' generated by the "v" command'); | ||
| console.log(' a - add the "current" utterance and intent labels to the model example set'); | ||
@@ -348,2 +358,3 @@ console.log(' r - remove the "current" utterance and intent labels from the model example set'); | ||
| console.log(`> Unknown-label threshold: ${this.unknownLabelPredictionThreshold}`); | ||
| console.log(`> Obfuscation flag: ${this.obfuscateEvaluationReport}`); | ||
| const labelResolverConfig = labelresolver_1.LabelResolver.getConfigJson(); | ||
@@ -469,2 +480,4 @@ console.log(`> Orchestrator configuration: ${labelResolverConfig}`); | ||
| // ---- NOTE ---- integrated step to produce analysis reports. | ||
| utility_1.Utility.toObfuscateLabelTextInReportUtility = this.obfuscateEvaluationReport; | ||
| utilitylabelresolver_1.UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver = this.obfuscateEvaluationReport; | ||
| utility_1.Utility.debuggingLog('OrchestratorPredict.commandLetV(), ready to call UtilityLabelResolver.resetLabelResolverSettingIgnoreSameExample("true")'); | ||
@@ -680,3 +693,3 @@ utilitylabelresolver_1.UtilityLabelResolver.resetLabelResolverSettingIgnoreSameExample(true); | ||
| async commandLetVUT(question) { | ||
| return this.commandLetVUTwithEntry(await question(OrchestratorPredict.questionForunknownLabelPredictionThreshold)); | ||
| return this.commandLetVUTwithEntry(await question(OrchestratorPredict.questionForUnknownLabelPredictionThreshold)); | ||
| } | ||
@@ -693,2 +706,11 @@ commandLetVUTwithEntry(entry) { | ||
| } | ||
| async commandLetVO(question) { | ||
| return this.commandLetVOwithEntry(await question(OrchestratorPredict.questionForObfuscateEvaluationReport)); | ||
| } | ||
| commandLetVOwithEntry(entry) { | ||
| const obfuscateEvaluationReportParameter = entry; | ||
| const obfuscateEvaluationReport = bf_dispatcher_2.Utility.toBoolean(obfuscateEvaluationReportParameter); | ||
| this.obfuscateEvaluationReport = obfuscateEvaluationReport; | ||
| return 0; | ||
| } | ||
| commandLetA() { | ||
@@ -777,3 +799,4 @@ const example = example_1.Example.newIntentExample(this.currentUtterance, this.currentIntentLabels); | ||
| OrchestratorPredict.questionForMultiLabelPredictionThreshold = 'Please enter a threshold for multi-label prediction > '; | ||
| OrchestratorPredict.questionForunknownLabelPredictionThreshold = 'Please enter a threshold for unknow label prediction > '; | ||
| OrchestratorPredict.questionForUnknownLabelPredictionThreshold = 'Please enter a threshold for unknow label prediction > '; | ||
| OrchestratorPredict.questionForObfuscateEvaluationReport = 'Please enter true/false for obfuscating lables/utterances in evaluation report > '; | ||
| //# sourceMappingURL=predict.js.map |
@@ -7,3 +7,3 @@ export declare class OrchestratorTest { | ||
| static readonly testingSetLabelsOutputFilename: string; | ||
| static runAsync(baseModelPath: string, inputPathConfiguration: string, testPathConfiguration: string, outputPath: string, ambiguousClosenessThresholdParameter: number, lowConfidenceScoreThresholdParameter: number, multiLabelPredictionThresholdParameter: number, unknownLabelPredictionThresholdParameter: number, fullEmbeddings?: boolean): Promise<void>; | ||
| static runAsync(baseModelPath: string, inputPathConfiguration: string, testPathConfiguration: string, outputPath: string, ambiguousClosenessThresholdParameter: number, lowConfidenceScoreThresholdParameter: number, multiLabelPredictionThresholdParameter: number, unknownLabelPredictionThresholdParameter: number, fullEmbeddings?: boolean, obfuscateEvaluationReport?: boolean): Promise<void>; | ||
| } |
@@ -13,8 +13,8 @@ "use strict"; | ||
| const orchestratorhelper_1 = require("./orchestratorhelper"); | ||
| const utility_1 = require("./utility"); | ||
| const utilitylabelresolver_1 = require("./utilitylabelresolver"); | ||
| const utility_1 = require("./utility"); | ||
| class OrchestratorTest { | ||
| // eslint-disable-next-line complexity | ||
| // eslint-disable-next-line max-params | ||
| static async runAsync(baseModelPath, inputPathConfiguration, testPathConfiguration, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings = false) { | ||
| static async runAsync(baseModelPath, inputPathConfiguration, testPathConfiguration, outputPath, ambiguousClosenessThresholdParameter, lowConfidenceScoreThresholdParameter, multiLabelPredictionThresholdParameter, unknownLabelPredictionThresholdParameter, fullEmbeddings = false, obfuscateEvaluationReport = false) { | ||
| // ----------------------------------------------------------------------- | ||
@@ -48,2 +48,5 @@ // ---- NOTE ---- process arguments | ||
| utility_1.Utility.debuggingLog(`fullEmbeddings=${fullEmbeddings}`); | ||
| utility_1.Utility.debuggingLog(`obfuscateEvaluationReport=${obfuscateEvaluationReport}`); | ||
| utility_1.Utility.toObfuscateLabelTextInReportUtility = obfuscateEvaluationReport; | ||
| utilitylabelresolver_1.UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver = obfuscateEvaluationReport; | ||
| // ----------------------------------------------------------------------- | ||
@@ -50,0 +53,0 @@ // ---- NOTE ---- load the snapshot set |
@@ -13,2 +13,3 @@ import { IConfusionMatrix } from '@microsoft/bf-dispatcher'; | ||
| static toPrintDetailedDebuggingLogToConsole: boolean; | ||
| static toObfuscateLabelTextInReportUtility: boolean; | ||
| static NumberOfInstancesPerProgressDisplayBatch: number; | ||
@@ -563,4 +564,4 @@ static readonly DefaultAmbiguousClosenessThresholdParameter: number; | ||
| }; | ||
| static selectedScoreStructureToHtmlTable(predictionScoreStructure: PredictionScoreStructure, unknownLabelPredictionThreshold: number, tableDescription?: string, selectedOutputDataArraryHeaders?: string[], outputDataColumnWidthSettings?: string[], indexes?: number[]): string; | ||
| static selectedScoreResultsToHtmlTable(scoreResultArray: Result[], indexes: number[], unknownLabelPredictionThreshold: number, tableDescription?: string, selectedOutputDataArraryHeaders?: string[], outputDataColumnWidthSettings?: string[]): string; | ||
| static selectedScoreStructureToHtmlTable(predictionScoreStructure: PredictionScoreStructure, unknownLabelPredictionThreshold: number, toObfuscateLabelTextInReport: boolean, tableDescription?: string, selectedOutputDataArraryHeaders?: string[], outputDataColumnWidthSettings?: string[], indexes?: number[]): string; | ||
| static selectedScoreResultsToHtmlTable(scoreResultArray: Result[], indexes: number[], unknownLabelPredictionThreshold: number, toObfuscateLabelTextInReport: boolean, tableDescription?: string, selectedOutputDataArraryHeaders?: string[], outputDataColumnWidthSettings?: string[]): string; | ||
| static evaluateMultiLabelSubsetPrediction(groundTruths: any[], predictions: any[]): number; | ||
@@ -567,0 +568,0 @@ static evaluateMultiLabelPrediction(groundTruths: any[], predictions: any[]): number[]; |
| import { PredictionScoreStructure } from './predictionscorestructure'; | ||
| export declare class UtilityLabelResolver { | ||
| static toObfuscateLabelTextInReportUtilityLabelResolver: boolean; | ||
| static resetLabelResolverSettingIgnoreSameExample(ignoreSameExample?: boolean, resetAll?: boolean): any; | ||
@@ -4,0 +5,0 @@ static resetLabelResolverSettingUseCompactEmbeddings(fullEmbeddings?: boolean, resetAll?: boolean): any; |
@@ -10,2 +10,3 @@ "use strict"; | ||
| const labeltype_1 = require("./labeltype"); | ||
| const label_1 = require("./label"); | ||
| const labelresolver_1 = require("./labelresolver"); | ||
@@ -69,4 +70,4 @@ const utility_1 = require("./utility"); | ||
| const labelsIndexes = labels.map((x) => utility_1.Utility.carefullyAccessStringMap(labelArrayAndMap.stringMap, x)); | ||
| const labelsConcatenated = utility_1.Utility.concatenateDataArrayToDelimitedString(labels); | ||
| const labelsConcatenatedToHtmlTable = utility_1.Utility.concatenateDataArrayToHtmlTable('label', labels); | ||
| const labelsConcatenated = utility_1.Utility.concatenateDataArrayToDelimitedString(labels.map((label) => label_1.Label.outputString(label, UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver))); | ||
| const labelsConcatenatedToHtmlTable = utility_1.Utility.concatenateDataArrayToHtmlTable('label', labels.map((label) => label_1.Label.outputString(label, UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver))); | ||
| if (utility_1.Utility.toPrintDetailedDebuggingLogToConsole) { | ||
@@ -109,4 +110,4 @@ utility_1.Utility.debuggingLog(`UtilityLabelResolver.score(), before calling score(), utterance=${utterance}`); | ||
| } | ||
| const labelsPredictedConcatenated = utility_1.Utility.concatenateDataArrayToDelimitedString(labelsPredicted); | ||
| const labelsPredictedConcatenatedToHtmlTable = utility_1.Utility.concatenateDataArrayToHtmlTable('label', labelsPredicted); | ||
| const labelsPredictedConcatenated = utility_1.Utility.concatenateDataArrayToDelimitedString(labelsPredicted.map((label) => label_1.Label.outputString(label, UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver))); | ||
| const labelsPredictedConcatenatedToHtmlTable = utility_1.Utility.concatenateDataArrayToHtmlTable('label', labelsPredicted.map((label) => label_1.Label.outputString(label, UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver))); | ||
| if (utility_1.Utility.toPrintDetailedDebuggingLogToConsole) { | ||
@@ -119,7 +120,7 @@ utility_1.Utility.debuggingLog(`UtilityLabelResolver.score(), JSON.stringify(labelsPredictedConcatenated)="${JSON.stringify(labelsPredictedConcatenated)}"`); | ||
| } | ||
| const predictedScoreStructureHtmlTable = utility_1.Utility.selectedScoreResultsToHtmlTable(scoreResultArray, labelsPredictedIndexes, unknownLabelPredictionThreshold, '', ['Label', 'Score', 'Closest Example'], ['30%', '10%', '60%']); | ||
| const predictedScoreStructureHtmlTable = utility_1.Utility.selectedScoreResultsToHtmlTable(scoreResultArray, labelsPredictedIndexes, unknownLabelPredictionThreshold, UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver, '', ['Label', 'Score', 'Closest Example'], ['30%', '10%', '60%']); | ||
| if (utility_1.Utility.toPrintDetailedDebuggingLogToConsole) { | ||
| utility_1.Utility.debuggingLog(`UtilityLabelResolver.score(), predictedScoreStructureHtmlTable="${predictedScoreStructureHtmlTable}"`); | ||
| } | ||
| const labelsScoreStructureHtmlTable = utility_1.Utility.selectedScoreResultsToHtmlTable(scoreResultArray, labels.map((x) => utility_1.Utility.carefullyAccessStringMap(labelArrayAndMap.stringMap, x)), unknownLabelPredictionThreshold, '', ['Label', 'Score', 'Closest Example'], ['30%', '10%', '60%']); | ||
| const labelsScoreStructureHtmlTable = utility_1.Utility.selectedScoreResultsToHtmlTable(scoreResultArray, labels.map((x) => utility_1.Utility.carefullyAccessStringMap(labelArrayAndMap.stringMap, x)), unknownLabelPredictionThreshold, UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver, '', ['Label', 'Score', 'Closest Example'], ['30%', '10%', '60%']); | ||
| if (utility_1.Utility.toPrintDetailedDebuggingLogToConsole) { | ||
@@ -165,2 +166,3 @@ utility_1.Utility.debuggingLog(`UtilityLabelResolver.score(), labelsScoreStructureHtmlTable="${labelsScoreStructureHtmlTable}"`); | ||
| exports.UtilityLabelResolver = UtilityLabelResolver; | ||
| UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver = true; | ||
| //# sourceMappingURL=utilitylabelresolver.js.map |
| { | ||
| "name": "@microsoft/bf-orchestrator", | ||
| "description": "APIs to interact with BF Orchestrator.", | ||
| "version": "4.11.0-beta.20201025.035cdcb", | ||
| "version": "4.11.0-beta.20201026.d8a7aef", | ||
| "author": "Microsoft", | ||
@@ -31,3 +31,3 @@ "bugs": "https://github.com/microsoft/botframework-cli/issues", | ||
| "@microsoft/bf-lu": "next", | ||
| "@microsoft/bf-dispatcher": "4.11.0-beta.20201025.035cdcb", | ||
| "@microsoft/bf-dispatcher": "4.11.0-beta.20201026.d8a7aef", | ||
| "@types/node-fetch": "~2.5.5", | ||
@@ -34,0 +34,0 @@ "node-fetch": "~2.6.0", |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by2.25%
710481
- Lines of code
- increased by1.42%
7877
Dependency changes
+ Added@microsoft/bf-dispatcher@4.11.0-beta.20201026.d8a7aef(transitive)
- Removed@microsoft/bf-dispatcher@4.11.0-beta.20201025.035cdcb(transitive)
Updated@microsoft/bf-dispatcher@4.11.0-beta.20201026.d8a7aef