Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | import {
Entity,
PrimaryGeneratedColumn,
Column,
BaseEntity,
ManyToOne,
Unique,
} from "typeorm"
import { Writable } from "stream"
import { User } from "./User"
import { Source } from "./Source"
import { Variable } from "./Variable"
import * as db from "../db"
import { arrToCsvRow, slugify } from "../../clientUtils/Util"
import filenamify from "filenamify"
@Entity("datasets")
@Unique(["name", "namespace"])
export class Dataset extends BaseEntity {
@PrimaryGeneratedColumn() id!: number
@Column() name!: string
@Column({ default: "owid" }) namespace!: string
@Column({ default: "" }) description!: string
@Column() createdAt!: Date
@Column() updatedAt!: Date
@Column() metadataEditedAt!: Date
@Column() metadataEditedByUserId!: number
@Column() dataEditedAt!: Date
@Column() dataEditedByUserId!: number
@Column({ default: false }) isPrivate!: boolean
@Column({ default: false }) nonRedistributable!: boolean
@ManyToOne(() => User, (user) => user.createdDatasets)
createdByUser!: User
// Export dataset variables to CSV (not including metadata)
static async writeCSV(datasetId: number, stream: Writable): Promise<void> {
const csvHeader = ["Entity", "Year"]
const variables = await db.queryMysql(
`SELECT name, id FROM variables v WHERE v.datasetId=? ORDER BY v.columnOrder ASC, v.id ASC`,
[datasetId]
)
for (const variable of variables) {
csvHeader.push(variable.name)
}
const columnIndexByVariableId: { [key: number]: number } = {}
for (const variable of variables) {
columnIndexByVariableId[variable.id] = csvHeader.indexOf(
variable.name
)
}
stream.write(arrToCsvRow(csvHeader))
const data = await db.queryMysql(
`
SELECT e.name AS entity, dv.year, dv.value, dv.variableId FROM data_values dv
JOIN variables v ON v.id=dv.variableId
JOIN datasets d ON v.datasetId=d.id
JOIN entities e ON dv.entityId=e.id
WHERE d.id=?
ORDER BY e.name ASC, dv.year ASC, v.columnOrder ASC, dv.variableId ASC`,
[datasetId]
)
let row: string[] = []
for (const datum of data) {
if (datum.entity !== row[0] || datum.year !== row[1]) {
// New row
if (row.length) {
stream.write(arrToCsvRow(row))
}
row = [datum.entity, datum.year]
for (const variable of variables) {
row.push("")
}
}
row[columnIndexByVariableId[datum.variableId]] = datum.value
}
// Final row
stream.write(arrToCsvRow(row))
stream.end()
}
static async setTags(datasetId: number, tagIds: number[]): Promise<void> {
await db.transaction(async (t) => {
const tagRows = tagIds.map((tagId) => [tagId, datasetId])
await t.execute(`DELETE FROM dataset_tags WHERE datasetId=?`, [
datasetId,
])
if (tagRows.length)
await t.execute(
`INSERT INTO dataset_tags (tagId, datasetId) VALUES ?`,
[tagRows]
)
})
}
async toCSV(): Promise<string> {
let csv = ""
await Dataset.writeCSV(this.id, {
write: (s: string) => (csv += s),
end: () => null,
} as any)
return csv
}
get filename(): string {
return filenamify(this.name)
}
get slug(): string {
return slugify(this.name)
}
// Return object representing datapackage.json for this dataset
async toDatapackage(): Promise<any> {
// XXX
const sources = await Source.find({ datasetId: this.id })
const variables = (await db
.knexTable(Variable.table)
.where({ datasetId: this.id })) as Variable.Row[]
const tags = await db.queryMysql(
`SELECT t.id, t.name FROM dataset_tags dt JOIN tags t ON t.id=dt.tagId WHERE dt.datasetId=?`,
[this.id]
)
const initialFields = [
{ name: "Entity", type: "string" },
{ name: "Year", type: "year" },
]
const dataPackage = {
name: this.name,
title: this.name,
id: this.id,
description:
(sources[0] &&
sources[0].description &&
sources[0].description.additionalInfo) ||
"",
sources: sources.map((s) => s.toDatapackage()),
owidTags: tags.map((t: any) => t.name),
resources: [
{
path: `${this.name}.csv`,
schema: {
fields: initialFields.concat(
variables.map((v) => ({
name: v.name,
type: "any",
description: v.description,
owidDisplaySettings: v.display,
}))
),
},
},
],
}
return dataPackage
}
}
|