parent
3d4f45d070
commit
e5d2d895d9
|
@ -0,0 +1,674 @@ |
||||
import React, { useState } from "react"; |
||||
import { |
||||
Button as ReactAdminButton, |
||||
useDataProvider, |
||||
useNotify, |
||||
Title, |
||||
} from "react-admin"; |
||||
import { parse as parseCsv, unparse as unparseCsv } from "papaparse"; |
||||
import GetAppIcon from "@material-ui/icons/GetApp"; |
||||
import { |
||||
Button, |
||||
Card, |
||||
CardActions, |
||||
CardContent, |
||||
CardHeader, |
||||
FormControlLabel, |
||||
Checkbox, |
||||
NativeSelect, |
||||
} from "@material-ui/core"; |
||||
import { useTranslate } from "ra-core"; |
||||
import Container from "@material-ui/core/Container/Container"; |
||||
import { generateRandomUser } from "./users"; |
||||
|
||||
const LOGGING = true; |
||||
|
||||
export const ImportButton = ({ label, variant = "text" }) => { |
||||
return ( |
||||
<ReactAdminButton |
||||
color="primary" |
||||
component="span" |
||||
variant={variant} |
||||
label={label} |
||||
> |
||||
<GetAppIcon style={{ transform: "rotate(180deg)", fontSize: "20" }} /> |
||||
</ReactAdminButton> |
||||
); |
||||
}; |
||||
|
||||
const expectedFields = ["id", "displayname"].sort(); |
||||
const optionalFields = [ |
||||
"user_type", |
||||
"guest", |
||||
"admin", |
||||
"deactivated", |
||||
"avatar_url", |
||||
"password", |
||||
].sort(); |
||||
|
||||
function TranslatableOption({ value, text }) { |
||||
const translate = useTranslate(); |
||||
return <option value={value}>{translate(text)}</option>; |
||||
} |
||||
|
||||
const FilePicker = props => { |
||||
const [values, setValues] = useState(null); |
||||
const [error, setError] = useState(null); |
||||
const [stats, setStats] = useState(null); |
||||
const [dryRun, setDryRun] = useState(true); |
||||
|
||||
const [progress, setProgress] = useState(null); |
||||
|
||||
const [importResults, setImportResults] = useState(null); |
||||
const [skippedRecords, setSkippedRecords] = useState(null); |
||||
|
||||
const [conflictMode, setConflictMode] = useState("stop"); |
||||
const [passwordMode, setPasswordMode] = useState(true); |
||||
const [useridMode, setUseridMode] = useState("ignore"); |
||||
|
||||
const translate = useTranslate(); |
||||
const notify = useNotify(); |
||||
|
||||
const dataProvider = useDataProvider(); |
||||
|
||||
const onFileChange = async e => { |
||||
if (progress !== null) return; |
||||
|
||||
setValues(null); |
||||
setError(null); |
||||
setStats(null); |
||||
setImportResults(null); |
||||
const file = e.target.files ? e.target.files[0] : null; |
||||
/* Let's refuse some unreasonably big files instead of freezing |
||||
* up the browser */ |
||||
if (file.size > 100000000) { |
||||
const message = translate("import_users.errors.unreasonably_big", { |
||||
size: (file.size / (1024 * 1024)).toFixed(2), |
||||
}); |
||||
notify(message); |
||||
setError(message); |
||||
return; |
||||
} |
||||
try { |
||||
parseCsv(file, { |
||||
header: true, |
||||
skipEmptyLines: true /* especially for a final EOL in the csv file */, |
||||
complete: result => { |
||||
if (result.error) { |
||||
setError(result.error); |
||||
} |
||||
/* Papaparse is very lenient, we may be able to salvage |
||||
* the data in the file. */ |
||||
verifyCsv(result, { setValues, setStats, setError }); |
||||
}, |
||||
}); |
||||
} catch { |
||||
setError(true); |
||||
return null; |
||||
} |
||||
}; |
||||
|
||||
const verifyCsv = ( |
||||
{ data, meta, errors }, |
||||
{ setValues, setStats, setError } |
||||
) => { |
||||
/* First, verify the presence of required fields */ |
||||
let eF = Array.from(expectedFields); |
||||
let oF = Array.from(optionalFields); |
||||
|
||||
meta.fields.forEach(name => { |
||||
if (eF.includes(name)) { |
||||
eF = eF.filter(v => v !== name); |
||||
} |
||||
if (oF.includes(name)) { |
||||
oF = oF.filter(v => v !== name); |
||||
} |
||||
}); |
||||
|
||||
if (eF.length !== 0) { |
||||
setError( |
||||
translate("import_users.error.required_field", { field: eF[0] }) |
||||
); |
||||
return false; |
||||
} |
||||
|
||||
// XXX after deciding on how "name" and friends should be handled below,
|
||||
// this place will want changes, too.
|
||||
|
||||
/* Collect some stats to prevent sneaky csv files from adding admin |
||||
users or something. |
||||
*/ |
||||
let stats = { |
||||
user_types: { default: 0 }, |
||||
is_guest: 0, |
||||
admin: 0, |
||||
deactivated: 0, |
||||
password: 0, |
||||
avatar_url: 0, |
||||
id: 0, |
||||
|
||||
total: data.length, |
||||
}; |
||||
|
||||
data.forEach((line, idx) => { |
||||
if (line.user_type === undefined || line.user_type === "") { |
||||
stats.user_types.default++; |
||||
} else { |
||||
stats.user_types[line.user_type] += 1; |
||||
} |
||||
/* XXX correct the csv export that react-admin offers for the users |
||||
* resource so it gives sensible field names and doesn't duplicate |
||||
* id as "name"? |
||||
*/ |
||||
if (meta.fields.includes("name")) { |
||||
delete line.name; |
||||
} |
||||
if (meta.fields.includes("user_type")) { |
||||
delete line.user_type; |
||||
} |
||||
if (meta.fields.includes("is_admin")) { |
||||
line.admin = line.is_admin; |
||||
delete line.is_admin; |
||||
} |
||||
|
||||
["is_guest", "admin", "deactivated"].forEach(f => { |
||||
if (line[f] === "true") { |
||||
stats[f]++; |
||||
line[f] = true; // we need true booleans instead of strings
|
||||
} else { |
||||
if (line[f] !== "false" && line[f] !== "") { |
||||
errors.push( |
||||
translate("import_users.error.invalid_value", { |
||||
field: f, |
||||
row: idx, |
||||
}) |
||||
); |
||||
} |
||||
line[f] = false; // default values to false
|
||||
} |
||||
}); |
||||
|
||||
if (line.password !== undefined && line.password !== "") { |
||||
stats.password++; |
||||
} |
||||
|
||||
if (line.avatar_url !== undefined && line.avatar_url !== "") { |
||||
stats.avatar_url++; |
||||
} |
||||
|
||||
if (line.id !== undefined && line.id !== "") { |
||||
stats.id++; |
||||
} |
||||
}); |
||||
|
||||
if (errors.length > 0) { |
||||
setError(errors); |
||||
} |
||||
setStats(stats); |
||||
setValues(data); |
||||
|
||||
return true; |
||||
}; |
||||
|
||||
const runImport = async e => { |
||||
if (progress !== null) { |
||||
notify("import_users.errors.already_in_progress"); |
||||
return; |
||||
} |
||||
|
||||
const results = await doImport( |
||||
dataProvider, |
||||
values, |
||||
conflictMode, |
||||
passwordMode, |
||||
useridMode, |
||||
dryRun, |
||||
setProgress, |
||||
setError |
||||
); |
||||
setImportResults(results); |
||||
// offer CSV download of skipped or errored records
|
||||
// (so that the user doesn't have to filter out successful
|
||||
// records manually when fixing stuff in the CSV)
|
||||
setSkippedRecords(unparseCsv(results.skippedRecords)); |
||||
if (LOGGING) console.log("Skipped records:"); |
||||
if (LOGGING) console.log(skippedRecords); |
||||
}; |
||||
|
||||
// XXX every single one of the requests will restart the activity indicator
|
||||
// which doesn't look very good.
|
||||
|
||||
const doImport = async ( |
||||
dataProvider, |
||||
data, |
||||
conflictMode, |
||||
passwordMode, |
||||
useridMode, |
||||
dryRun, |
||||
setProgress, |
||||
setError |
||||
) => { |
||||
let skippedRecords = []; |
||||
let erroredRecords = []; |
||||
let succeededRecords = []; |
||||
let changeStats = { |
||||
toAdmin: 0, |
||||
toGuest: 0, |
||||
toRegular: 0, |
||||
replacedPassword: 0, |
||||
}; |
||||
let entriesDone = 0; |
||||
let entriesCount = data.length; |
||||
try { |
||||
setProgress({ done: entriesDone, limit: entriesCount }); |
||||
for (const entry of data) { |
||||
let userRecord = {}; |
||||
let overwriteData = {}; |
||||
// No need to do a bunch of cryptographic random number getting if
|
||||
// we are using neither a generated password nor a generated user id.
|
||||
if ( |
||||
useridMode === "ignore" || |
||||
entry.id === undefined || |
||||
entry.password === undefined || |
||||
passwordMode === false |
||||
) { |
||||
overwriteData = generateRandomUser(); |
||||
// Ignoring IDs or the entry lacking an ID means we keep the
|
||||
// ID field in the overwrite data.
|
||||
if (!(useridMode === "ignore" || entry.id === undefined)) { |
||||
delete overwriteData.id; |
||||
} |
||||
|
||||
// Not using passwords from the csv or this entry lacking a password
|
||||
// means we keep the password field in the overwrite data.
|
||||
if ( |
||||
!( |
||||
passwordMode === false || |
||||
entry.password === undefined || |
||||
entry.password === "" |
||||
) |
||||
) { |
||||
delete overwriteData.password; |
||||
} |
||||
} |
||||
/* TODO record update stats (especially admin no -> yes, deactivated x -> !x, ... */ |
||||
Object.assign(userRecord, entry); |
||||
Object.assign(userRecord, overwriteData); |
||||
|
||||
/* For these modes we will consider the ID that's in the record. |
||||
* If the mode is "stop", we will not continue adding more records, and |
||||
* we will offer information on what was already added and what was |
||||
* skipped. |
||||
* |
||||
* If the mode is "skip", we record the record for later, but don't |
||||
* send it to the server. |
||||
* |
||||
* If the mode is "update", we change fields that are reasonable to |
||||
* update. |
||||
* - If the "password mode" is "true" (i.e. "use passwords from csv"): |
||||
* - if the record has a password |
||||
* - send the password along with the record |
||||
* - if the record has no password |
||||
* - generate a new password |
||||
* - If the "password mode" is "false" |
||||
* - never generate a new password to update existing users with |
||||
*/ |
||||
|
||||
/* We just act as if there are no IDs in the CSV, so every user will be |
||||
* created anew. |
||||
* We do a simple retry loop so that an accidental hit on an existing ID |
||||
* doesn't trip us up. |
||||
*/ |
||||
if (LOGGING) |
||||
console.log( |
||||
"will check for existence of record " + JSON.stringify(userRecord) |
||||
); |
||||
let retries = 0; |
||||
const submitRecord = recordData => { |
||||
return dataProvider.getOne("users", { id: recordData.id }).then( |
||||
async alreadyExists => { |
||||
if (LOGGING) console.log("already existed"); |
||||
|
||||
if (useridMode === "update" || conflictMode === "skip") { |
||||
skippedRecords.push(recordData); |
||||
} else if (conflictMode === "stop") { |
||||
throw new Error( |
||||
translate("import_users.error.id_exits", { |
||||
id: recordData.id, |
||||
}) |
||||
); |
||||
} else { |
||||
const overwriteData = generateRandomUser(); |
||||
const newRecordData = Object.assign({}, recordData, { |
||||
id: overwriteData.id, |
||||
}); |
||||
retries++; |
||||
if (retries > 512) { |
||||
console.warn("retry loop got stuck? pathological situation?"); |
||||
skippedRecords.push(recordData); |
||||
} else { |
||||
await submitRecord(newRecordData); |
||||
} |
||||
} |
||||
}, |
||||
async okToSubmit => { |
||||
if (LOGGING) |
||||
console.log( |
||||
"OK to create record " + |
||||
recordData.id + |
||||
" (" + |
||||
recordData.displayname + |
||||
")." |
||||
); |
||||
|
||||
if (!dryRun) { |
||||
await dataProvider.create("users", { data: recordData }); |
||||
} |
||||
succeededRecords.push(recordData); |
||||
} |
||||
); |
||||
}; |
||||
|
||||
await submitRecord(userRecord); |
||||
entriesDone++; |
||||
setProgress({ done: entriesDone, limit: data.length }); |
||||
} |
||||
|
||||
setProgress(null); |
||||
} catch (e) { |
||||
setError( |
||||
translate("import_users.error.at_entry", { |
||||
entry: entriesDone + 1, |
||||
message: e.message, |
||||
}) |
||||
); |
||||
setProgress(null); |
||||
} |
||||
return { |
||||
skippedRecords, |
||||
erroredRecords, |
||||
succeededRecords, |
||||
totalRecordCount: entriesCount, |
||||
changeStats, |
||||
wasDryRun: dryRun, |
||||
}; |
||||
}; |
||||
|
||||
const downloadSkippedRecords = () => { |
||||
const element = document.createElement("a"); |
||||
console.log(skippedRecords); |
||||
const file = new Blob([skippedRecords], { |
||||
type: "text/comma-separated-values", |
||||
}); |
||||
element.href = URL.createObjectURL(file); |
||||
element.download = "skippedRecords.csv"; |
||||
document.body.appendChild(element); // Required for this to work in FireFox
|
||||
element.click(); |
||||
}; |
||||
|
||||
const onConflictModeChanged = async e => { |
||||
if (progress !== null) { |
||||
return; |
||||
} |
||||
|
||||
const value = e.target.value; |
||||
setConflictMode(value); |
||||
}; |
||||
|
||||
const onPasswordModeChange = e => { |
||||
if (progress !== null) { |
||||
return; |
||||
} |
||||
|
||||
setPasswordMode(e.target.checked); |
||||
}; |
||||
|
||||
const onUseridModeChanged = async e => { |
||||
if (progress !== null) { |
||||
return; |
||||
} |
||||
|
||||
const value = e.target.value; |
||||
setUseridMode(value); |
||||
}; |
||||
|
||||
const onDryRunModeChanged = ev => { |
||||
if (progress !== null) { |
||||
return; |
||||
} |
||||
setDryRun(ev.target.checked); |
||||
}; |
||||
|
||||
// render individual small components
|
||||
|
||||
const statsCards = stats && |
||||
!importResults && [ |
||||
<Container> |
||||
<CardHeader |
||||
title={translate("import_users.cards.importstats.header")} |
||||
/> |
||||
<CardContent> |
||||
<div> |
||||
{translate( |
||||
"import_users.cards.importstats.users_total", |
||||
stats.total |
||||
)} |
||||
</div> |
||||
<div> |
||||
{translate( |
||||
"import_users.cards.importstats.guest_count", |
||||
stats.is_guest |
||||
)} |
||||
</div> |
||||
<div> |
||||
{translate( |
||||
"import_users.cards.importstats.admin_count", |
||||
stats.admin |
||||
)} |
||||
</div> |
||||
</CardContent> |
||||
</Container>, |
||||
<Container> |
||||
<CardHeader title={translate("import_users.cards.ids.header")} /> |
||||
<CardContent> |
||||
<div> |
||||
{stats.id === stats.total |
||||
? translate("import_users.cards.ids.all_ids_present") |
||||
: translate("import_users.cards.ids.count_ids_present", stats.id)} |
||||
</div> |
||||
{stats.id > 0 ? ( |
||||
<div> |
||||
<NativeSelect |
||||
onChange={onUseridModeChanged} |
||||
value={useridMode} |
||||
enabled={(progress !== null).toString()} |
||||
> |
||||
<TranslatableOption |
||||
value="ignore" |
||||
text="import_users.cards.ids.mode.ignore" |
||||
/> |
||||
<TranslatableOption |
||||
value="update" |
||||
text="import_users.cards.ids.mode.update" |
||||
/> |
||||
</NativeSelect> |
||||
</div> |
||||
) : ( |
||||
"" |
||||
)} |
||||
</CardContent> |
||||
</Container>, |
||||
<Container> |
||||
<CardHeader title={translate("import_users.cards.passwords.header")} /> |
||||
<CardContent> |
||||
<div> |
||||
{stats.password === stats.total |
||||
? translate("import_users.cards.passwords.all_passwords_present") |
||||
: translate( |
||||
"import_users.cards.passwords.count_passwords_present", |
||||
stats.password |
||||
)} |
||||
</div> |
||||
{stats.password > 0 ? ( |
||||
<div> |
||||
<FormControlLabel |
||||
control={ |
||||
<Checkbox |
||||
checked={passwordMode} |
||||
enabled={(progress !== null).toString()} |
||||
onChange={onPasswordModeChange} |
||||
/> |
||||
} |
||||
label={translate("import_users.cards.passwords.use_passwords")} |
||||
/> |
||||
</div> |
||||
) : ( |
||||
"" |
||||
)} |
||||
</CardContent> |
||||
</Container>, |
||||
]; |
||||
|
||||
let conflictCards = stats && !importResults && ( |
||||
<Container> |
||||
<CardHeader title={translate("import_users.cards.conflicts.header")} /> |
||||
<CardContent> |
||||
<div> |
||||
<NativeSelect |
||||
onChange={onConflictModeChanged} |
||||
value={conflictMode} |
||||
enabled={(progress !== null).toString()} |
||||
> |
||||
<TranslatableOption |
||||
value="stop" |
||||
text="import_users.cards.conflicts.mode.stop" |
||||
/> |
||||
<TranslatableOption |
||||
value="skip" |
||||
text="import_users.cards.conflicts.mode.skip" |
||||
/> |
||||
</NativeSelect> |
||||
</div> |
||||
</CardContent> |
||||
</Container> |
||||
); |
||||
|
||||
let errorCards = error && ( |
||||
<Container> |
||||
<CardHeader title={translate("import_users.error.error")} /> |
||||
<CardContent> |
||||
{(Array.isArray(error) ? error : [error]).map(e => ( |
||||
<div>{e}</div> |
||||
))} |
||||
</CardContent> |
||||
</Container> |
||||
); |
||||
|
||||
let uploadCard = !importResults && ( |
||||
<Container> |
||||
<CardHeader title={translate("import_users.cards.upload.header")} /> |
||||
<CardContent> |
||||
{translate("import_users.cards.upload.explanation")} |
||||
<a href="/data/example.csv">example.csv</a> |
||||
<br /> |
||||
<br /> |
||||
<input |
||||
type="file" |
||||
onChange={onFileChange} |
||||
enabled={(progress !== null).toString()} |
||||
/> |
||||
</CardContent> |
||||
</Container> |
||||
); |
||||
|
||||
let resultsCard = importResults && ( |
||||
<CardContent> |
||||
<CardHeader title={translate("import_users.cards.results.header")} /> |
||||
<div> |
||||
{translate( |
||||
"import_users.cards.results.total", |
||||
importResults.totalRecordCount |
||||
)} |
||||
<br /> |
||||
{translate( |
||||
"import_users.cards.results.successful", |
||||
importResults.succeededRecords.length |
||||
)} |
||||
<br /> |
||||
{importResults.skippedRecords.length |
||||
? [ |
||||
translate( |
||||
"import_users.cards.results.skipped", |
||||
importResults.skippedRecords.length |
||||
), |
||||
<div> |
||||
<button onClick={downloadSkippedRecords}> |
||||
{translate("import_users.cards.results.download_skipped")} |
||||
</button> |
||||
</div>, |
||||
<br />, |
||||
] |
||||
: ""} |
||||
{importResults.erroredRecords.length |
||||
? [ |
||||
translate( |
||||
"import_users.cards.results.skipped", |
||||
importResults.erroredRecords.length |
||||
), |
||||
<br />, |
||||
] |
||||
: ""} |
||||
<br /> |
||||
{importResults.wasDryRun && [ |
||||
translate("import_users.cards.results.simulated_only"), |
||||
<br />, |
||||
]} |
||||
</div> |
||||
</CardContent> |
||||
); |
||||
|
||||
let startImportCard = |
||||
!values || values.length === 0 || importResults ? undefined : ( |
||||
<CardActions> |
||||
<FormControlLabel |
||||
control={ |
||||
<Checkbox |
||||
checked={dryRun} |
||||
onChange={onDryRunModeChanged} |
||||
enabled={(progress !== null).toString()} |
||||
/> |
||||
} |
||||
label={translate("import_users.cards.startImport.simulate_only")} |
||||
/> |
||||
<Button |
||||
size="large" |
||||
onClick={runImport} |
||||
enabled={(progress !== null).toString()} |
||||
> |
||||
{translate("import_users.cards.startImport.run_import")} |
||||
</Button> |
||||
{progress !== null ? ( |
||||
<div> |
||||
{progress.done} of {progress.limit} done |
||||
</div> |
||||
) : null} |
||||
</CardActions> |
||||
); |
||||
|
||||
let allCards = []; |
||||
if (uploadCard) allCards.push(uploadCard); |
||||
if (errorCards) allCards.push(errorCards); |
||||
if (conflictCards) allCards.push(conflictCards); |
||||
if (statsCards) allCards.push(...statsCards); |
||||
if (startImportCard) allCards.push(startImportCard); |
||||
if (resultsCard) allCards.push(resultsCard); |
||||
|
||||
let cardContainer = <Card>{allCards}</Card>; |
||||
|
||||
return [ |
||||
<Title defaultTitle={translate("import_users.title")} />, |
||||
cardContainer, |
||||
]; |
||||
}; |
||||
|
||||
export const ImportFeature = FilePicker; |
||||
@ -0,0 +1,39 @@ |
||||
// in src/Menu.js
|
||||
import * as React from "react"; |
||||
import { useSelector } from "react-redux"; |
||||
import { useMediaQuery } from "@material-ui/core"; |
||||
import { MenuItemLink, getResources } from "react-admin"; |
||||
import DefaultIcon from "@material-ui/icons/ViewList"; |
||||
import LabelIcon from "@material-ui/icons/Label"; |
||||
|
||||
const Menu = ({ onMenuClick, logout }) => { |
||||
const isXSmall = useMediaQuery(theme => theme.breakpoints.down("xs")); |
||||
const open = useSelector(state => state.admin.ui.sidebarOpen); |
||||
const resources = useSelector(getResources); |
||||
return ( |
||||
<div> |
||||
{resources.map(resource => ( |
||||
<MenuItemLink |
||||
key={resource.name} |
||||
to={`/${resource.name}`} |
||||
primaryText={ |
||||
(resource.options && resource.options.label) || resource.name |
||||
} |
||||
leftIcon={resource.icon ? <resource.icon /> : <DefaultIcon />} |
||||
onClick={onMenuClick} |
||||
sidebarIsOpen={open} |
||||
/> |
||||
))} |
||||
<MenuItemLink |
||||
to="/custom-route" |
||||
primaryText="Miscellaneous" |
||||
leftIcon={<LabelIcon />} |
||||
onClick={onMenuClick} |
||||
sidebarIsOpen={open} |
||||
/> |
||||
{isXSmall && logout} |
||||
</div> |
||||
); |
||||
}; |
||||
|
||||
export default Menu; |
||||
Loading…
Reference in new issue