Skip to content

Commit

Permalink
support multiple possible file names (#546)
Browse files Browse the repository at this point in the history 
* almost works

* no await in loop

* only calculate fileName once

* remove fileNames from doUpload (buggy with requesting multiple file names), fix "not found" temporarily

* some clean up

* more

* rest

* this isn't used anywhere

* less nest

* const

* small clean ups

* const here too
  • Loading branch information
@sshane
sshane authored Jul 25, 2024
1 parent 6d0e6fe commit 79880f1
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 28 deletions.
53 changes: 32 additions & 21 deletions src/actions/files.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ import * as Types from './types';
import { deviceOnCellular, getDeviceFromState, deviceVersionAtLeast, asyncSleep } from '../utils';

export const FILE_NAMES = {
qcameras: 'qcamera.ts',
cameras: 'fcamera.hevc',
dcameras: 'dcamera.hevc',
ecameras: 'ecamera.hevc',
qlogs: 'qlog.bz2',
logs: 'rlog.bz2',
qcameras: ['qcamera.ts'],
cameras: ['fcamera.hevc'],
dcameras: ['dcamera.hevc'],
ecameras: ['ecamera.hevc'],
qlogs: ['qlog.bz2', 'qlog.zst'],
logs: ['rlog.bz2', 'rlog.zst'],
};
const MAX_OPEN_REQUESTS = 15;
const MAX_RETRIES = 5;
Expand All @@ -21,7 +21,7 @@ let openRequests = 0;

function pathToFileName(dongleId, path) {
const [seg, fileType] = path.split('/');
const type = Object.entries(FILE_NAMES).find((e) => e[1] === fileType)[0];
const type = Object.entries(FILE_NAMES).find((e) => e[1].includes(fileType))[0];
return `${dongleId}|${seg}/${type}`;
}

Expand Down Expand Up @@ -169,7 +169,7 @@ export function fetchUploadQueue(dongleId) {
const segNum = urlParts[urlParts.length - 2];
const datetime = urlParts[urlParts.length - 3];
const dongle = urlParts[urlParts.length - 4];
const type = Object.entries(FILE_NAMES).find((e) => e[1] === filename)[0];
const type = Object.entries(FILE_NAMES).find((e) => e[1].includes(filename))[0];
const fileName = `${dongle}|${datetime}--${segNum}/${type}`;
const waitingWifi = Boolean(deviceOnCellular(device) && uploading.allow_cellular === false);
uploadingFiles[fileName] = {
Expand Down Expand Up @@ -207,7 +207,7 @@ export function fetchUploadQueue(dongleId) {
};
}

export function doUpload(dongleId, fileNames, paths, urls) {
export function doUpload(dongleId, paths, urls) {
return async (dispatch, getState) => {
const { device } = getState();
let loopedUploads = !deviceVersionAtLeast(device, '0.8.13');
Expand All @@ -230,26 +230,37 @@ export function doUpload(dongleId, fileNames, paths, urls) {
&& resp.error.data.message === 'too many values to unpack (expected 3)') {
loopedUploads = true;
} else if (!resp || resp.error) {
const newUploading = fileNames.reduce((state, fn) => {
state[fn] = {};
const newUploading = paths.reduce((state, path) => {
state[pathToFileName(dongleId, path)] = {};
return state;
}, {});
dispatch(updateDeviceOnline(dongleId, Math.floor(Date.now() / 1000)));
dispatch(updateFiles(newUploading));
} else if (resp.offline) {
dispatch(updateDeviceOnline(dongleId, 0));
} else if (resp.result === 'Device offline, message queued') {
const newUploading = fileNames.reduce((state, fn) => {
state[fn] = { progress: 0, current: false };
const newUploading = paths.reduce((state, path) => {
state[pathToFileName(dongleId, path)] = { progress: 0, current: false };
return state;
}, {});
dispatch(updateFiles(newUploading));
} else if (resp.result) {
if (resp.result.failed) {
const uploading = resp.result.failed
.filter((path) => paths.indexOf(path) > -1)
let failed = resp.result.failed || [];

// only if all file names for a segment file type failed
let failedFiltered = [];
for (const f of failed) {
let failedCnt = failed.filter((p) => pathToFileName(dongleId, p) === pathToFileName(dongleId, f)).length;
let requestedCnt = paths.filter((p) => pathToFileName(dongleId, p) === pathToFileName(dongleId, f)).length;
if (failedCnt >= requestedCnt) {
failedFiltered.push(f);
}
}

if (failedFiltered) {
const uploading = failedFiltered
.reduce((state, path) => {
const fn = fileNames[paths.indexOf(path)];
const fn = pathToFileName(dongleId, path);
state[fn] = { notFound: true };
return state;
}, {});
Expand All @@ -260,7 +271,7 @@ export function doUpload(dongleId, fileNames, paths, urls) {
}

if (loopedUploads) {
for (let i = 0; i < fileNames.length; i++) {
for (let i = 0; i < paths.length; i++) {
const payload = {
id: 0,
jsonrpc: '2.0',
Expand All @@ -272,18 +283,18 @@ export function doUpload(dongleId, fileNames, paths, urls) {
const resp = await athenaCall(dongleId, payload, 'files_actions_athena_upload');
if (!resp || resp.error) {
const uploading = {};
uploading[fileNames[i]] = {};
uploading[pathToFileName(dongleId, paths[i])] = {};
dispatch(updateDeviceOnline(dongleId, Math.floor(Date.now() / 1000)));
dispatch(updateFiles(uploading));
} else if (resp.offline) {
dispatch(updateDeviceOnline(dongleId, 0));
} else if (resp.result === 'Device offline, message queued') {
const uploading = {};
uploading[fileNames[i]] = { progress: 0, current: false };
uploading[pathToFileName(dongleId, paths[i])] = { progress: 0, current: false };
dispatch(updateFiles(uploading));
} else if (resp.result === 404 || resp?.result?.failed?.[0] === paths[i]) {
const uploading = {};
uploading[fileNames[i]] = { notFound: true };
uploading[pathToFileName(dongleId, paths[i])] = { notFound: true };
dispatch(updateFiles(uploading));
} else if (resp.result) {
dispatch(fetchUploadQueue(dongleId));
Expand Down
21 changes: 15 additions & 6 deletions src/components/DriveView/Media.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,16 +334,25 @@ class Media extends Component {
}));

const routeNoDongleId = currentRoute.fullname.split('|')[1];
const path = `${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${FILE_NAMES[type]}`;
const fileName = `${dongleId}|${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${type}`;

const uploading = {};
uploading[fileName] = { requested: true };
this.props.dispatch(updateFiles(uploading));

const urls = await fetchUploadUrls(dongleId, [path]);
let paths = [];
let url_promises = [];

// request all possible file names
for (const fn of FILE_NAMES[type]) {
const path = `${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${fn}`;
paths.push(path);
url_promises.push(fetchUploadUrls(dongleId, [path]).then(urls => urls[0]));
}

const urls = await Promise.all(url_promises);
if (urls) {
this.props.dispatch(doUpload(dongleId, [fileName], [path], urls));
this.props.dispatch(doUpload(dongleId, paths, urls));
}
}

Expand Down Expand Up @@ -376,14 +385,14 @@ class Media extends Component {
}
this.props.dispatch(updateFiles(uploading));

const paths = Object.keys(uploading).map((fileName) => {
const paths = Object.keys(uploading).flatMap((fileName) => {
const [seg, type] = fileName.split('/');
return `${seg.split('|')[1]}/${FILE_NAMES[type]}`;
return FILE_NAMES[type].map(file => `${seg.split('|')[1]}/${file}`);
});

const urls = await fetchUploadUrls(dongleId, paths);
if (urls) {
this.props.dispatch(doUpload(dongleId, Object.keys(uploading), paths, urls));
this.props.dispatch(doUpload(dongleId, paths, urls));
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/components/Files/UploadQueue.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ class UploadQueue extends Component {
</div>
</td>
<td className={ classes.uploadCell } style={ cellStyle }>
{ FILE_NAMES[type].split('.')[0].substring(0, logNameLength) }
{ FILE_NAMES[type][0].split('.')[0].substring(0, logNameLength) }
</td>
{ upload.current
? (
Expand Down

0 comments on commit 79880f1

Please sign in to comment.