super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

处理后台新需求
Build Backend JAR / build (push) Failing after 16m58s
Details

Browse Source
This commit is contained in:
supernijia
2026-08-09 20:08:39 +08:00
parent f7e26ee266
commit 38216a1885
42 changed files with 3425 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/blueprints/admin_api.py
+135 -66
View File
@@ -334,7 +334,8 @@ class _PermissionProxyError(Exception):
self.status = status
def _proxy_permission_java(method, path, *, params=None, json_data=None, current_row=None):
def _proxy_permission_java(
method, path, *, params=None, json_data=None, files=None, data=None, current_row=None):
"""Call Java permission APIs using either forwarded JWT or trusted Flask identity."""
proxy_params = {}
request_row = getattr(g, '_current_user_row', None) if has_request_context() else None
@@ -353,6 +354,8 @@ def _proxy_permission_java(method, path, *, params=None, json_data=None, current
path,
params=proxy_params or None,
json_data=json_data,
files=files,
data=data,
headers=headers,
)
return result, error_response, status
@@ -905,7 +908,7 @@ def list_users():
return jsonify({'success': False, 'error': '需要登录'}), 403
page = max(1, int(request.args.get('page', 1)))
_, _, denied = _ensure_backend_menu_access(
'users', 'history', 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'users', 'history', 'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
@@ -1335,7 +1338,12 @@ def _parse_admin_datetime_arg(name):
raise ValueError(f'{name} 时间格式无效') from exc
_SHOP_DATA_CRAWL_ADMIN_COLUMNS = """
_SHOP_DATA_CRAWL_LATEST_TIME_SQL = (
'COALESCE(df.last_success_at, df.updated_at, t.finished_at, t.updated_at, t.created_at)'
)
_SHOP_DATA_CRAWL_ADMIN_COLUMNS = f"""
r.id AS result_id, r.task_id, r.user_id, r.source_filename AS shop_name,
r.source_file_url AS shop_id, r.result_filename, r.result_file_url,
r.result_file_size, r.result_content_type, r.row_count,
@@ -1343,6 +1351,7 @@ _SHOP_DATA_CRAWL_ADMIN_COLUMNS = """
r.created_at AS result_created_at,
t.task_no, t.status AS task_status, t.request_json, t.result_json,
t.error_message AS task_error, t.created_at, t.updated_at, t.finished_at,
{_SHOP_DATA_CRAWL_LATEST_TIME_SQL} AS latest_file_updated_at,
u.username,
(SELECT j.id FROM biz_task_file_job j
WHERE j.module_type = 'SHOP_DATA_CRAWL' AND j.result_id = r.id
@@ -1446,16 +1455,18 @@ def _shop_data_crawl_admin_item(row, group_names=None):
def _shop_data_crawl_group_item(group_row, result_rows, group_names):
"""Build one shop group and cap its children to the newest three results."""
"""Build one shop group and expose its current daily workbook only."""
raw_shop_name = group_row.get('shop_name') or ''
display_shop_name = raw_shop_name or '未命名'
group_key = _shop_data_crawl_shop_key(raw_shop_name)
children = result_rows.get(group_key)
if children is None:
children = result_rows.get(str(raw_shop_name).strip(), [])
children = children[:3]
children = children[:1]
result_items = [_shop_data_crawl_admin_item(row, group_names) for row in children]
latest_created_at = group_row.get('latest_created_at')
if latest_created_at is None and children:
latest_created_at = children[0].get('latest_file_updated_at')
if latest_created_at is None and result_items:
latest_created_at = result_items[0].get('created_at')
return {
@@ -1541,6 +1552,7 @@ def list_shop_data_crawl_tasks():
grouped_from_sql = (
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql
)
@@ -1554,7 +1566,8 @@ def list_shop_data_crawl_tasks():
total = int((cur.fetchone() or {}).get('total') or 0)
cur.execute(
'SELECT ' + shop_key_sql + ' AS shop_name, MAX(t.created_at) AS latest_created_at'
'SELECT ' + shop_key_sql + ' AS shop_name, MAX('
+ _SHOP_DATA_CRAWL_LATEST_TIME_SQL + ') AS latest_created_at'
+ grouped_from_sql +
' GROUP BY ' + shop_key_sql +
' ORDER BY latest_created_at DESC, shop_name ASC LIMIT %s OFFSET %s',
@@ -1570,14 +1583,16 @@ def list_shop_data_crawl_tasks():
cur.execute(
'SELECT ranked.* FROM (SELECT ' + _SHOP_DATA_CRAWL_ADMIN_COLUMNS +
', ROW_NUMBER() OVER (PARTITION BY ' + shop_key_sql +
' ORDER BY t.created_at DESC, r.id DESC) AS shop_row_number '
' ORDER BY ' + _SHOP_DATA_CRAWL_LATEST_TIME_SQL
+ ' DESC, r.id DESC) AS shop_row_number '
' FROM biz_file_result r '
'JOIN biz_file_task t ON t.id = r.task_id '
'LEFT JOIN biz_shop_data_crawl_daily_file df ON df.latest_result_id = r.id '
'LEFT JOIN users u ON u.id = r.user_id '
'WHERE ' + where_sql +
f' AND {shop_key_sql} IN ({placeholders})' +
') ranked WHERE ranked.shop_row_number <= 3 '
'ORDER BY ranked.created_at DESC, ranked.result_id DESC',
') ranked WHERE ranked.shop_row_number <= 1 '
'ORDER BY ranked.latest_file_updated_at DESC, ranked.result_id DESC',
tuple(params + selected_shop_names),
)
for row in cur.fetchall():
@@ -2863,6 +2878,26 @@ def delete_shop_key(item_id):
# ---------- 数据去重总数据 ----------
def _parse_positive_group_id(raw_value):
try:
group_id = int(str(raw_value or '').strip())
except (TypeError, ValueError):
return None
return group_id if group_id > 0 else None
def _format_dedupe_total_data_item(item):
return {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'group_id': item.get('groupId'),
'group_name': item.get('groupName') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
@admin_api.route('/dedupe-total-data')
@login_required
def list_dedupe_total_data():
@@ -2873,6 +2908,7 @@ def list_dedupe_total_data():
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
keyword = (request.args.get('keyword') or '').strip()
username = (request.args.get('username') or '').strip()
group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId'))
start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip()
end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip()
params = {
@@ -2886,22 +2922,19 @@ def list_dedupe_total_data():
params['startDate'] = start_date
if end_date:
params['endDate'] = end_date
data, error_response, status = _proxy_backend_java(
if group_id is not None:
params['groupId'] = group_id
data, error_response, status = _proxy_permission_java(
'GET',
'/api/admin/dedupe-total-data',
params=params,
current_row=current_row,
)
if error_response is not None:
return error_response, status
payload = data.get('data') or {}
items = [
{
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
_format_dedupe_total_data_item(item)
for item in (payload.get('items') or [])
]
return jsonify({
@@ -2929,10 +2962,18 @@ def export_dedupe_total_data():
params['startDate'] = start_date
if end_date:
params['endDate'] = end_date
group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId'))
if group_id is not None:
params['groupId'] = group_id
url = f"{backend_java_base_url}/api/admin/dedupe-total-data/export"
try:
resp = _get_backend_java_session().get(url, params=params, timeout=60)
resp = _get_backend_java_session().get(
url,
params=params,
headers={'X-Internal-Token': _resolve_internal_token()},
timeout=60,
)
except requests.RequestException:
return jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502
if resp.status_code >= 400:
@@ -2964,10 +3005,10 @@ def dedupe_total_data_import_progress(import_id):
_, current_row, denied = _ensure_dedupe_total_data_access()
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'GET',
f'/api/admin/dedupe-total-data/import/{import_id}',
params={'operatorId': current_row.get('id')},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -2995,14 +3036,18 @@ def import_dedupe_total_data():
file_storage = request.files.get('file')
if not file_storage or file_storage.filename == '':
return jsonify({'success': False, 'error': '请选择 Excel 文件'})
group_id = _parse_positive_group_id(request.form.get('group_id') or request.form.get('groupId'))
if group_id is None:
return jsonify({'success': False, 'error': '请选择分组'})
files = {
'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream')
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/dedupe-total-data/import',
files=files,
data={'operatorId': current_row.get('id')},
data={'groupId': group_id},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3020,10 +3065,10 @@ def dedupe_total_data_delete_import_progress(import_id):
_, current_row, denied = _ensure_dedupe_total_data_access()
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'GET',
f'/api/admin/dedupe-total-data/delete-import/{import_id}',
params={'operatorId': current_row.get('id')},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3051,14 +3096,18 @@ def delete_import_dedupe_total_data():
file_storage = request.files.get('file')
if not file_storage or file_storage.filename == '':
return jsonify({'success': False, 'error': '请选择 Excel 文件'})
group_id = _parse_positive_group_id(request.form.get('group_id') or request.form.get('groupId'))
if group_id is None:
return jsonify({'success': False, 'error': '请选择分组'})
files = {
'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream')
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/dedupe-total-data/delete-import',
files=files,
data={'operatorId': current_row.get('id')},
data={'groupId': group_id},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3077,12 +3126,15 @@ def create_dedupe_total_data():
if denied:
return denied
data = request.get_json() or {}
payload = {'dataValue': (data.get('data_value') or '').strip()}
result, error_response, status = _proxy_backend_java(
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'groupId': _parse_positive_group_id(data.get('group_id') or data.get('groupId')),
}
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/dedupe-total-data',
params={'operatorId': current_row.get('id')},
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3090,13 +3142,7 @@ def create_dedupe_total_data():
return jsonify({
'success': True,
'msg': result.get('message') or '创建成功',
'item': {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
},
'item': _format_dedupe_total_data_item(item),
})
@@ -3107,12 +3153,15 @@ def update_dedupe_total_data(item_id):
if denied:
return denied
data = request.get_json() or {}
payload = {'dataValue': (data.get('data_value') or '').strip()}
result, error_response, status = _proxy_backend_java(
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'groupId': _parse_positive_group_id(data.get('group_id') or data.get('groupId')),
}
result, error_response, status = _proxy_permission_java(
'PUT',
f'/api/admin/dedupe-total-data/{item_id}',
params={'operatorId': current_row.get('id')},
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3120,13 +3169,7 @@ def update_dedupe_total_data(item_id):
return jsonify({
'success': True,
'msg': result.get('message') or '更新成功',
'item': {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'uploader_user_id': item.get('uploaderUserId'),
'username': item.get('username') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
},
'item': _format_dedupe_total_data_item(item),
})
@@ -3136,10 +3179,10 @@ def delete_dedupe_total_data(item_id):
_, current_row, denied = _ensure_dedupe_total_data_access()
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'DELETE',
f'/api/admin/dedupe-total-data/{item_id}',
params={'operatorId': current_row.get('id')},
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3157,23 +3200,36 @@ def _format_invalid_asin_data_item(item):
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'brand': item.get('brand') or '',
'group_id': item.get('groupId'),
'group_name': item.get('groupName') or '',
'record_source': item.get('recordSource') or 'AUTO',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
@admin_api.route('/invalid-asin-data')
@admin_required
@login_required
def list_invalid_asin_data():
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
page = max(1, int(request.args.get('page', 1)))
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
keyword = (request.args.get('keyword') or '').strip()
data, error_response, status = _proxy_backend_java(
group_id_raw = (request.args.get('group_id') or '').strip()
group_id = int(group_id_raw) if group_id_raw.isdigit() and int(group_id_raw) > 0 else None
params = {
'page': page,
'pageSize': page_size,
'keyword': keyword,
}
if group_id is not None:
params['groupId'] = group_id
data, error_response, status = _proxy_permission_java(
'GET',
'/api/admin/invalid-asin-data',
params={'page': page, 'pageSize': page_size, 'keyword': keyword},
params=params,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3189,20 +3245,22 @@ def list_invalid_asin_data():
@admin_api.route('/invalid-asin-data', methods=['POST'])
@admin_required
@login_required
def create_invalid_asin_data():
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
data = request.get_json() or {}
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'brand': (data.get('brand') or '').strip(),
'groupId': data.get('group_id'),
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'POST',
'/api/admin/invalid-asin-data',
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3215,20 +3273,22 @@ def create_invalid_asin_data():
@admin_api.route('/invalid-asin-data/<int:item_id>', methods=['PUT'])
@admin_required
@login_required
def update_invalid_asin_data(item_id):
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
data = request.get_json() or {}
payload = {
'dataValue': (data.get('data_value') or '').strip(),
'brand': (data.get('brand') or '').strip(),
'groupId': data.get('group_id'),
}
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'PUT',
f'/api/admin/invalid-asin-data/{item_id}',
json_data=payload,
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3241,14 +3301,15 @@ def update_invalid_asin_data(item_id):
@admin_api.route('/invalid-asin-data/<int:item_id>', methods=['DELETE'])
@admin_required
@login_required
def delete_invalid_asin_data(item_id):
_, _, denied = _ensure_admin_menu_access('invalid-asin-data')
role, current_row, denied = _ensure_backend_menu_access('invalid-asin-data')
if denied:
return denied
result, error_response, status = _proxy_backend_java(
result, error_response, status = _proxy_permission_java(
'DELETE',
f'/api/admin/invalid-asin-data/{item_id}',
current_row=current_row,
)
if error_response is not None:
return error_response, status
@@ -3515,21 +3576,29 @@ def delete_shop_manage(item_id):
@login_required
def list_shop_manage_groups():
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
groups, error_response, status = _load_expanded_shop_manage_groups(role, current_row)
if error_response is not None:
return error_response, status
return jsonify({'success': True, 'items': [_format_shop_manage_group_item(item) for item in (groups or [])]})
items = [_format_shop_manage_group_item(item) for item in (groups or [])]
locked_group_id = None
if role != 'super_admin':
locked_group_id = next((item.get('id') for item in items if item.get('id')), None)
return jsonify({
'success': True,
'items': items,
'locked_group_id': locked_group_id,
})
@admin_api.route('/shop-manage-group', methods=['POST'])
@login_required
def create_shop_manage_group():
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
@@ -3576,7 +3645,7 @@ def create_shop_manage_group():
@login_required
def update_shop_manage_group(item_id):
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied
@@ -3823,7 +3892,7 @@ def delete_skip_price_asin_country(item_id, country):
@login_required
def delete_shop_manage_group(item_id):
role, current_row, denied = _ensure_backend_menu_access(
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data'
'shop-manage', 'skip-price-asin', 'query-asin', 'dedupe-total-data', 'invalid-asin-data'
)
if denied:
return denied