Skip to content

Feature/cleansing enhancements #421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hhhhsc701 merged 2 commits into from
Mar 11, 2026
Merged

Feature/cleansing enhancements #421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
759cd43
支持处理文件下载
hhhhsc701 Mar 10, 2026
f90453a
创建任务增加筛选
hhhhsc701 Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions deployment/helm/datamate/charts/frontend/templates/_helpers.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ Selector labels
{{- define "frontend.selectorLabels" -}}
app.kubernetes.io/name: {{ include "frontend.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app: {{ include "frontend.name" . }}
{{- end }}

{{/*
Expand Down
53 changes: 24 additions & 29 deletions frontend/src/i18n/locales/en/common.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1331,6 +1331,7 @@
"pause": "Pause",
"compare": "Compare",
"download": "Download",
"batchDownload": "Batch Download",
"delete": "Delete",
"batchDownload": "Batch Download",
"edit": "Edit",
Expand All @@ -1345,6 +1346,10 @@
"operatorOrchestration": "Operator Orchestration"
},
"task": {
"sections": {
"taskInfo": "Task Information",
"dataSourceSelection": "Data Source Selection"
},
"columns": {
"taskName": "Task Name",
"taskId": "Task ID",
Expand All @@ -1361,35 +1366,23 @@
"dataSizeChange": "Data Size Change",
"actions": "Actions"
},
"form": {
"name": "Name",
"namePlaceholder": "Enter task name",
"description": "Description",
"descriptionPlaceholder": "Describe the task goals and requirements",
"srcDataset": "Source Dataset",
"srcDatasetPlaceholder": "Please select source dataset",
"destDatasetName": "Target Dataset Name",
"destDatasetNamePlaceholder": "Enter or select target dataset name",
"destDatasetType": "Target Dataset Type",
"destDatasetTypeRequired": "Please select target dataset type",
"useSourceDataset": "Use Source Dataset",
"useSourceDatasetHint": "When checked, target dataset will use source dataset and cannot be modified",
"destDatasetNameRequired": "Please enter target dataset name",
"cannotUseSourceDataset": "Cannot use source dataset as target. Check 'Use Source Dataset' or enter a different name"
},
"sections": {
"taskInfo": "Task Info",
"dataSourceSelection": "Data Source Selection"
},
"messages": {
"taskCreated": "Task created successfully",
"taskStarted": "Task started successfully",
"taskPaused": "Task paused successfully",
"taskDeleted": "Task deleted successfully",
"taskDetailFailed": "Failed to fetch task details",
"fetchTaskResultFailed": "Failed to fetch task results",
"fetchTaskLogFailed": "Failed to fetch task logs"
}
"form": {
"name": "Name",
"namePlaceholder": "Enter task name",
"description": "Description",
"descriptionPlaceholder": "Describe the purpose",
"srcDataset": "Source Dataset",
"srcDatasetPlaceholder": "Select source dataset",
"destDatasetType": "Target Dataset Type",
"useSourceDataset": "Use Source Dataset",
"useSourceDatasetHint": "Output will be replaced in source dataset",
"cannotUseSourceDataset": "Cannot use source dataset as destination",
"destDatasetNameRequired": "Destination dataset name is required",
"destDatasetTypeRequired": "Target dataset type is required",
"allTypes": "All",
"showingDatasets": "Showing {{count}} datasets",
"noDatasetsFound": "No datasets found"
}
},
"template": {
"columns": {
Expand Down Expand Up @@ -1439,8 +1432,10 @@
"searchFileName": "Search file name",
"searchFileType": "Search file type",
"selectFiles": "{{count}} files selected",
"selectedFilesCount": "Selected {{count}} files",
"notCompleted": "Not Completed",
"downloadNotAvailable": "Not Available Yet",
"downloadFilesFailed": "Failed to download files",
"compareDialogTitle": "File Comparison - {{fileName}}",
"beforeProcessing": "Before Processing",
"afterProcessing": "After Processing",
Expand Down
56 changes: 26 additions & 30 deletions frontend/src/i18n/locales/zh/common.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1331,8 +1331,8 @@
"pause": "暂停",
"compare": "对比",
"download": "下载",
"delete": "删除",
"batchDownload": "批量下载",
"delete": "删除",
"edit": "编辑",
"selectAll": "全选",
"search": "搜索",
Expand All @@ -1345,6 +1345,10 @@
"operatorOrchestration": "算子编排"
},
"task": {
"sections": {
"taskInfo": "任务信息",
"dataSourceSelection": "数据源选择"
},
"columns": {
"taskName": "任务名称",
"taskId": "任务ID",
Expand All @@ -1361,35 +1365,25 @@
"dataSizeChange": "数据量变化",
"actions": "操作"
},
"form": {
"name": "名称",
"namePlaceholder": "输入任务名称",
"description": "描述",
"descriptionPlaceholder": "描述任务的目标和要求",
"srcDataset": "源数据集",
"srcDatasetPlaceholder": "请选择源数据集",
"destDatasetName": "目标数据集名称",
"destDatasetNamePlaceholder": "输入或选择目标数据集名称",
"destDatasetType": "目标数据集类型",
"destDatasetTypeRequired": "请选择目标数据集类型",
"useSourceDataset": "选择源数据集",
"useSourceDatasetHint": "勾选后目标数据集将使用源数据集，不可修改",
"destDatasetNameRequired": "请输入目标数据集名称",
"cannotUseSourceDataset": "不能使用源数据集作为目标数据集，请勾选\"选择源数据集\"或输入其他名称"
},
"sections": {
"taskInfo": "任务信息",
"dataSourceSelection": "数据源选择"
},
"messages": {
"taskCreated": "任务已创建",
"taskStarted": "任务已启动",
"taskPaused": "任务已暂停",
"taskDeleted": "任务已删除",
"taskDetailFailed": "获取任务详情失败",
"fetchTaskResultFailed": "获取数据处理结果失败",
"fetchTaskLogFailed": "获取数据处理日志失败"
}
"form": {
"name": "名称",
"namePlaceholder": "输入任务名称",
"description": "描述",
"descriptionPlaceholder": "描述任务的目标和要求",
"srcDataset": "源数据集",
"srcDatasetPlaceholder": "请选择源数据集",
"destDatasetName": "目标数据集名称",
"destDatasetNamePlaceholder": "输入或选择目标数据集名称",
"destDatasetType": "目标数据集类型",
"destDatasetTypeRequired": "请选择目标数据集类型",
"useSourceDataset": "选择源数据集",
"useSourceDatasetHint": "勾选后目标数据集将使用源数据集，不可修改",
"destDatasetNameRequired": "请输入目标数据集名称",
"cannotUseSourceDataset": "不能使用源数据集作为目标数据集，请勾选\"选择源数据集\"或输入其他名称",
"allTypes": "全部",
"showingDatasets": "显示 {{count}} 个数据集",
"noDatasetsFound": "未找到匹配的数据集"
}
},
"template": {
"columns": {
Expand Down Expand Up @@ -1439,8 +1433,10 @@
"searchFileName": "搜索文件名",
"searchFileType": "搜索文件类型",
"selectFiles": "已选择 {{count}} 个文件",
"selectedFilesCount": "已选择 {{count}} 个文件",
"notCompleted": "未完成",
"downloadNotAvailable": "暂未开放",
"downloadFilesFailed": "下载文件失败",
"compareDialogTitle": "文件对比 - {{fileName}}",
"beforeProcessing": "处理前",
"afterProcessing": "处理后",
Expand Down
74 changes: 46 additions & 28 deletions frontend/src/pages/DataCleansing/Create/components/CreateTaskStepOne.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ export default function CreateTaskStepOne({
let dataset = null;
if (key === "srcDatasetId") {
dataset = datasets.find((d) => d.id === value);
// 如果勾选了"选择源数据集"，自动更新目标数据集名称
const newDestName = useSourceDataset ? (dataset?.name || "") : allValues.destDatasetName;
form.setFieldValue("destDatasetName", newDestName);
setTaskConfig({
Expand Down Expand Up @@ -94,7 +93,6 @@ export default function CreateTaskStepOne({
}
};

// 过滤掉当前选中的源数据集（当不勾选"选择源数据集"时）
const getFilteredDatasetOptions = () => {
const srcDatasetId = form.getFieldValue("srcDatasetId");
if (useSourceDataset || !srcDatasetId) {
Expand All @@ -120,25 +118,47 @@ export default function CreateTaskStepOne({
<h2 className="font-medium text-gray-900 pt-6 mb-2 text-base">
{t("dataCleansing.task.sections.dataSourceSelection")}
</h2>

<Form.Item label={t("dataCleansing.task.form.srcDataset")} name="srcDatasetId" required>
<Select
placeholder={t("dataCleansing.task.form.srcDatasetPlaceholder")}
options={datasets.map((dataset) => {
return {
label: (
<div className="flex items-center justify-between gap-3 py-2">
<div className="flex items-center font-sm text-gray-900">
<span className="mr-2">{dataset.icon}</span>
<span>{dataset.name}</span>
</div>
<div className="text-xs text-gray-500">{dataset.size}</div>
showSearch
labelRender={(option) => {
const dataset = datasets.find(d => d.id === option.value);
return dataset?.name || option.label;
}}
filterOption={(input, option) => {
const dataset = datasets.find(d => d.id === option?.value);
if (!dataset) return false;
const searchLower = input.toLowerCase();
return dataset.name.toLowerCase().includes(searchLower) ||
(dataset.description?.toLowerCase().includes(searchLower) ?? false);
}}
options={getFilteredDatasetOptions().map((dataset) => ({
label: dataset.name,
value: dataset.id,
}))}
optionRender={(option) => {
const dataset = datasets.find(d => d.id === option.value);
if (!dataset) return null;
return (
<div className="flex items-center justify-between gap-3 py-1">
<div className="flex items-center text-gray-900">
<span className="mr-2">{dataset.icon}</span>
<span>{dataset.name}</span>
</div>
),
value: dataset.id,
};
})}
<div className="text-xs text-gray-500">{dataset.size}</div>
</div>
);
}}
notFoundContent={
<div className="text-center py-4 text-gray-400">
{t("dataCleansing.task.form.noDatasetsFound")}
</div>
}
/>
</Form.Item>

<div className="flex items-center gap-1 mb-1">
<span className="text-red-500">*</span>
<label className="text-sm text-gray-700 mr-4">{t("dataCleansing.task.form.destDatasetName")}</label>
Expand Down Expand Up @@ -175,20 +195,18 @@ export default function CreateTaskStepOne({
]}
>
<AutoComplete
options={getFilteredDatasetOptions().map((dataset) => {
return {
label: (
<div className="flex items-center justify-between gap-3 py-2">
<div className="flex items-center font-sm text-gray-900">
<span className="mr-2">{dataset.icon}</span>
<span>{dataset.name}</span>
</div>
<div className="text-xs text-gray-500">{dataset.size}</div>
options={getFilteredDatasetOptions().map((dataset) => ({
label: (
<div className="flex items-center justify-between gap-3 py-2">
<div className="flex items-center font-sm text-gray-900">
<span className="mr-2">{dataset.icon}</span>
<span>{dataset.name}</span>
</div>
),
value: dataset.name,
};
})}
<div className="text-xs text-gray-500">{dataset.size}</div>
</div>
),
value: dataset.name,
}))}
filterOption={(inputValue, option) => {
return option.value.toLowerCase().startsWith(inputValue.toLowerCase());
}}
Expand Down
14 changes: 11 additions & 3 deletions frontend/src/pages/DataCleansing/Create/components/OperatorOrchestration.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,22 @@ const OperatorFlow: React.FC<OperatorFlowProps> = ({
<Select
placeholder={t("dataCleansing.operatorOrchestration.selectTemplate")}
className="min-w-64"
showSearch
filterOption={(input, option) => {
const template = templates.find(t => t.id === option?.value);
if (!template) return false;
const searchLower = input.toLowerCase();
return template.name.toLowerCase().includes(searchLower) ||
(template.description?.toLowerCase().includes(searchLower) ?? false);
}}
options={templates}
value={currentTemplate?.value}
value={currentTemplate?.id}
onChange={(value) =>
setCurrentTemplate(
templates.find((t) => t.value === value) || null
templates.find((t) => t.id === value) || null
)
}
></Select>
/>
</div>
</div>
{/* 编排区域 */}
Expand Down
59 changes: 39 additions & 20 deletions frontend/src/pages/DataCleansing/Detail/components/FileTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,25 @@ export default function FileTable({result, fetchTaskResult}) {
setSelectedFile(file);
setShowFileCompareDialog(true);
};
const handleBatchDownload = () => {
// 实际下载逻辑
const handleDownload = async () => {
try {
const response = await fetch(`/api/cleaning/tasks/${id}/result/download`);
if (!response.ok) {
throw new Error('Download failed');
}
const blob = await response.blob();
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `task_${id}_files.zip`;
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
} catch (error) {
console.error('Download failed:', error);
message.error(t("dataCleansing.detail.fileTable.downloadFilesFailed"));
}
};

function formatFileSize(bytes: number, decimals: number = 2): string {
Expand Down Expand Up @@ -332,7 +349,7 @@ export default function FileTable({result, fetchTaskResult}) {
{
title: t("dataCleansing.detail.fileTable.actions"),
key: "action",
width: 200,
width: 150,
render: (_text: string, record: any) => (
<div className="flex">
{record.status === "COMPLETED" ? (
Expand All @@ -352,9 +369,13 @@ export default function FileTable({result, fetchTaskResult}) {
{t("dataCleansing.actions.compare")}
</Button>
)}
<Popover content={t("dataCleansing.detail.fileTable.downloadNotAvailable")}>
<Button type="link" size="small" disabled>{t("dataCleansing.actions.download")}</Button>
</Popover>
<Button
type="link"
size="small"
onClick={() => handleDownload()}
>
{t("dataCleansing.actions.download")}
</Button>
</div>
),
},
Expand All @@ -363,20 +384,18 @@ export default function FileTable({result, fetchTaskResult}) {
return (
<>
{selectedFileIds.length > 0 && (
<div className="mb-4 flex justify-between">
<div className="flex items-center gap-2">
<span className="text-sm text-gray-600">
{t("dataCleansing.detail.fileTable.downloadNotAvailable", {count: selectedFileIds.length})}
</span>
<Button
onClick={handleBatchDownload}
size="small"
type="primary"
icon={<Download className="w-4 h-4 mr-2" />}
>
{t("dataCleansing.actions.batchDownload")}
</Button>
</div>
<div className="mb-4 flex justify-between items-center">
<span className="text-sm text-gray-600">
{t("dataCleansing.detail.fileTable.selectedFilesCount", { count: selectedFileIds.length })}
</span>
<Button
onClick={handleDownload}
size="small"
type="primary"
icon={<Download className="w-4 h-4 mr-2" />}
>
{t("dataCleansing.actions.download")}
</Button>
</div>
)}
<Table
Expand Down
Loading
Loading