Skip to content

Commit

Permalink
Merge pull request #1441 from AyushAgrawal-A2/xlsx_formula
Browse files Browse the repository at this point in the history 
feat: import formulas when importing xlsx file
  • Loading branch information
@davidkircos
davidkircos committed Jul 8, 2024
2 parents c4cc983 + 524a710 commit 08a90d3
Show file tree
Hide file tree
Showing 16 changed files with 745 additions and 113 deletions.
36 changes: 30 additions & 6 deletions quadratic-client/src/app/gridGL/cells/CellsArray.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { sheets } from '@/app/grid/controller/Sheets';
import { Sheet } from '@/app/grid/sheet/Sheet';
import { inlineEditorHandler } from '@/app/gridGL/HTMLGrid/inlineEditor/inlineEditorHandler';
import { Coordinate } from '@/app/gridGL/types/size';
import { JsRenderCodeCell } from '@/app/quadratic-core-types';
import { JsCodeCell, JsRenderCodeCell, RunError } from '@/app/quadratic-core-types';
import mixpanel from 'mixpanel-browser';
import { Container, Graphics, ParticleContainer, Point, Rectangle, Sprite, Texture } from 'pixi.js';
import { colors } from '../../theme/colors';
import { dashedTextures } from '../dashedTextures';
Expand Down Expand Up @@ -64,14 +65,37 @@ export class CellsArray extends Container {
}
};

private updateCodeCell = (options: { sheetId: string; x: number; y: number; renderCodeCell?: JsRenderCodeCell }) => {
if (options.sheetId === this.cellsSheet.sheetId) {
if (options.renderCodeCell) {
this.codeCells.set(this.key(options.x, options.y), options.renderCodeCell);
private updateCodeCell = (options: {
sheetId: string;
x: number;
y: number;
renderCodeCell?: JsRenderCodeCell;
codeCell?: JsCodeCell;
}) => {
const { sheetId, x, y, renderCodeCell, codeCell } = options;
if (sheetId === this.cellsSheet.sheetId) {
if (renderCodeCell) {
this.codeCells.set(this.key(x, y), renderCodeCell);
} else {
this.codeCells.delete(this.key(options.x, options.y));
this.codeCells.delete(this.key(x, y));
}
this.create();

if (!!codeCell && codeCell.std_err !== null && codeCell.evaluation_result) {
try {
// std_err is not null, so evaluation_result will be RunError
const runError = JSON.parse(codeCell.evaluation_result) as RunError;
// track unimplemented errors
if (typeof runError.msg === 'object' && 'Unimplemented' in runError.msg) {
mixpanel.track('[CellsArray].updateCodeCell', {
type: codeCell.language,
error: runError.msg,
});
}
} catch (error) {
console.error('[CellsArray] Error parsing codeCell.evaluation_result', error);
}
}
}
};

Expand Down
2 changes: 1 addition & 1 deletion quadratic-client/src/app/quadratic-core-types/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export type Axis = "X" | "Y";
export interface Instant { seconds: number, }
export interface Duration { years: number, months: number, seconds: number, }
export interface RunError { span: Span | null, msg: RunErrorMsg, }
export type RunErrorMsg = { "PythonError": string } | "Spill" | "Unimplemented" | "UnknownError" | { "InternalError": string } | { "Unterminated": string } | { "Expected": { expected: string, got: string | null, } } | { "Unexpected": string } | { "TooManyArguments": { func_name: string, max_arg_count: number, } } | { "MissingRequiredArgument": { func_name: string, arg_name: string, } } | "BadFunctionName" | "BadCellReference" | "BadNumber" | { "ExactArraySizeMismatch": { expected: ArraySize, got: ArraySize, } } | { "ExactArrayAxisMismatch": { axis: Axis, expected: number, got: number, } } | { "ArrayAxisMismatch": { axis: Axis, expected: number, got: number, } } | "EmptyArray" | "NonRectangularArray" | "NonLinearArray" | "ArrayTooBig" | "CircularReference" | "Overflow" | "DivideByZero" | "NegativeExponent" | "NotANumber" | "Infinity" | "IndexOutOfBounds" | "NoMatch" | "InvalidArgument";
export type RunErrorMsg = { "PythonError": string } | "Spill" | { "Unimplemented": string } | "UnknownError" | { "InternalError": string } | { "Unterminated": string } | { "Expected": { expected: string, got: string | null, } } | { "Unexpected": string } | { "TooManyArguments": { func_name: string, max_arg_count: number, } } | { "MissingRequiredArgument": { func_name: string, arg_name: string, } } | "BadFunctionName" | "BadCellReference" | "BadNumber" | { "ExactArraySizeMismatch": { expected: ArraySize, got: ArraySize, } } | { "ExactArrayAxisMismatch": { axis: Axis, expected: number, got: number, } } | { "ArrayAxisMismatch": { axis: Axis, expected: number, got: number, } } | "EmptyArray" | "NonRectangularArray" | "NonLinearArray" | "ArrayTooBig" | "CircularReference" | "Overflow" | "DivideByZero" | "NegativeExponent" | "NotANumber" | "Infinity" | "IndexOutOfBounds" | "NoMatch" | "InvalidArgument";
export interface Pos { x: bigint, y: bigint, }
export interface Rect { min: Pos, max: Pos, }
export interface Span { start: number, end: number, }
Expand Down
5 changes: 3 additions & 2 deletions quadratic-core/src/controller/execution/execute_operation/execute_code.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,9 @@ impl GridController {
op: Operation,
) {
if let Operation::ComputeCode { sheet_pos } = op {
if !transaction.is_user() {
unreachable!("Only a user transaction should have a ComputeCode");
if !transaction.is_user() && !transaction.is_server() {
dbgjs!("Only a user/server transaction should have a ComputeCode");
return;
}
let sheet_id = sheet_pos.sheet_id;
let Some(sheet) = self.try_sheet(sheet_id) else {
Expand Down
10 changes: 6 additions & 4 deletions quadratic-core/src/controller/execution/run_code/get_cells.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use uuid::Uuid;
use crate::{
controller::{execution::TransactionType, GridController},
error_core::CoreError,
Rect,
Rect, RunError, RunErrorMsg,
};
use serde::{Deserialize, Serialize};

Expand Down Expand Up @@ -52,12 +52,14 @@ impl GridController {
} else {
// unable to find sheet by name, generate error
let mut msg = format!("Sheet '{}' not found", sheet_name);

if let Some(line_number) = line_number {
msg = format!("{} at line {}", msg, line_number);
}

let error = match self.code_cell_sheet_error(&mut transaction, &msg, line_number) {
let run_error = RunError {
span: None,
msg: RunErrorMsg::PythonError(msg.clone().into()),
};
let error = match self.code_cell_sheet_error(&mut transaction, &run_error) {
Ok(_) => CoreError::CodeCellSheetError(msg.to_owned()),
Err(err) => err,
};
Expand Down
23 changes: 9 additions & 14 deletions quadratic-core/src/controller/execution/run_code/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,8 +242,7 @@ impl GridController {
pub(super) fn code_cell_sheet_error(
&mut self,
transaction: &mut PendingTransaction,
error_msg: &str,
line_number: Option<u32>,
error: &RunError,
) -> Result<()> {
let sheet_pos = match transaction.current_sheet_pos {
Some(sheet_pos) => sheet_pos,
Expand All @@ -266,18 +265,12 @@ impl GridController {
// cell may have been deleted before the async operation completed
return Ok(());
};
if !matches!(code_cell, CellValue::Code(_)) {
let CellValue::Code(code_cell_value) = code_cell else {
// code may have been replaced while waiting for async operation
return Ok(());
}
};

let msg = RunErrorMsg::PythonError(error_msg.to_owned().into());
let span = line_number.map(|line_number| Span {
start: line_number,
end: line_number,
});
let error = RunError { span, msg };
let result = CodeRunResult::Err(error);
let result = CodeRunResult::Err(error.clone());

let new_code_run = match sheet.code_run(pos) {
Some(old_code_run) => {
Expand All @@ -288,7 +281,7 @@ impl GridController {
line_number: old_code_run.line_number,
output_type: old_code_run.output_type.clone(),
std_out: None,
std_err: Some(error_msg.to_owned()),
std_err: Some(error.msg.to_string()),
spill_error: false,
last_modified: Utc::now(),

Expand All @@ -300,10 +293,12 @@ impl GridController {
formatted_code_string: None,
result,
return_type: None,
line_number,
line_number: error
.span
.map(|span| span.line_number_of_str(&code_cell_value.code) as u32),
output_type: None,
std_out: None,
std_err: Some(error_msg.to_owned()),
std_err: Some(error.msg.to_string()),
spill_error: false,
last_modified: Utc::now(),
cells_accessed: transaction.cells_accessed.clone(),
Expand Down
57 changes: 22 additions & 35 deletions quadratic-core/src/controller/execution/run_code/run_formula.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,42 +17,29 @@ impl GridController {
let mut ctx = Ctx::new(self.grid(), sheet_pos);
transaction.current_sheet_pos = Some(sheet_pos);
match parse_formula(&code, sheet_pos.into()) {
Ok(parsed) => {
match parsed.eval(&mut ctx, false) {
Ok(value) => {
transaction.cells_accessed = ctx.cells_accessed;
let new_code_run = CodeRun {
std_out: None,
std_err: None,
formatted_code_string: None,
spill_error: false,
last_modified: Utc::now(),
cells_accessed: transaction.cells_accessed.clone(),
result: CodeRunResult::Ok(value),
return_type: None,
line_number: None,
output_type: None,
};
self.finalize_code_run(transaction, sheet_pos, Some(new_code_run), None);
}
Err(error) => {
let msg = error.msg.to_string();
let line_number = error.span.map(|span| span.start);

// todo: propagate the result
let _ = self.code_cell_sheet_error(
transaction,
&msg,
// todo: span should be multiline
line_number,
);
}
Ok(parsed) => match parsed.eval(&mut ctx, false) {
Ok(value) => {
transaction.cells_accessed = ctx.cells_accessed;
let new_code_run = CodeRun {
std_out: None,
std_err: None,
formatted_code_string: None,
spill_error: false,
last_modified: Utc::now(),
cells_accessed: transaction.cells_accessed.clone(),
result: CodeRunResult::Ok(value),
return_type: None,
line_number: None,
output_type: None,
};
self.finalize_code_run(transaction, sheet_pos, Some(new_code_run), None);
}
}
Err(e) => {
let msg = e.to_string();
// todo: propagate the result
let _ = self.code_cell_sheet_error(transaction, &msg, None);
Err(error) => {
let _ = self.code_cell_sheet_error(transaction, &error);
}
},
Err(error) => {
let _ = self.code_cell_sheet_error(transaction, &error);
}
}
}
Expand Down
71 changes: 51 additions & 20 deletions quadratic-core/src/controller/operations/import.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use lexicon_fractional_index::key_between;
use crate::{
cell_values::CellValues,
controller::GridController,
grid::{file::sheet_schema::export_sheet, Sheet, SheetId},
CellValue, Pos, SheetPos,
grid::{file::sheet_schema::export_sheet, CodeCellLanguage, Sheet, SheetId},
CellValue, CodeCellValue, Pos, SheetPos,
};
use bytes::Bytes;
use calamine::{Data as ExcelData, Reader as ExcelReader, Xlsx, XlsxError};
Expand Down Expand Up @@ -130,28 +130,32 @@ impl GridController {
file_name: &str,
) -> Result<Vec<Operation>> {
let mut ops = vec![] as Vec<Operation>;
let insert_at = Pos::default();
let error =
|message: String| anyhow!("Error parsing Excel file {}: {}", file_name, message);
let error = |e: XlsxError| anyhow!("Error parsing Excel file {file_name}: {e}");

let cursor = Cursor::new(file);
let mut workbook: Xlsx<_> =
ExcelReader::new(cursor).map_err(|e: XlsxError| error(e.to_string()))?;
let mut workbook: Xlsx<_> = ExcelReader::new(cursor).map_err(error)?;
let sheets = workbook.sheet_names().to_owned();

// first cell in excel is A1, but first cell in quadratic is A0
// so we need to offset rows by 1, so that values are inserted in the original A1 notations cell
// this is required so that cell references (A1 notations) in formulas are correct
let xlsx_range_to_pos = |(row, col)| Pos {
x: col as i64,
y: row as i64 + 1,
};

let mut order = key_between(&None, &None).unwrap_or("A0".to_string());
for sheet_name in sheets {
// add the sheet
let mut sheet = Sheet::new(SheetId::new(), sheet_name.to_owned(), order.clone());
order = key_between(&Some(order), &None).unwrap_or("A0".to_string());

let range = workbook
.worksheet_range(&sheet_name)
.map_err(|e: XlsxError| error(e.to_string()))?;

// values
let range = workbook.worksheet_range(&sheet_name).map_err(error)?;
let insert_at = range.start().map_or_else(Pos::default, xlsx_range_to_pos);
for (y, row) in range.rows().enumerate() {
for (x, col) in row.iter().enumerate() {
let cell_value = match col {
for (x, cell) in row.iter().enumerate() {
let cell_value = match cell {
ExcelData::Empty => continue,
ExcelData::String(value) => CellValue::Text(value.to_string()),
ExcelData::DateTimeIso(ref value) => CellValue::Text(value.to_string()),
Expand Down Expand Up @@ -191,12 +195,36 @@ impl GridController {
);
}
}

// formulas
let formula = workbook.worksheet_formula(&sheet_name).map_err(error)?;
let insert_at = formula.start().map_or_else(Pos::default, xlsx_range_to_pos);
let mut formula_compute_ops = vec![];
for (y, row) in formula.rows().enumerate() {
for (x, cell) in row.iter().enumerate() {
if !cell.is_empty() {
let pos = Pos {
x: insert_at.x + x as i64,
y: insert_at.y + y as i64,
};
let cell_value = CellValue::Code(CodeCellValue {
language: CodeCellLanguage::Formula,
code: cell.to_string(),
});
sheet.set_cell_value(pos, cell_value);
// add code compute operation, to generate code runs
formula_compute_ops.push(Operation::ComputeCode {
sheet_pos: pos.to_sheet_pos(sheet.id),
});
}
}
}
// add new sheets
ops.push(Operation::AddSheetSchema {
schema: export_sheet(&sheet),
});
ops.extend(formula_compute_ops);
}

Ok(ops)
}

Expand Down Expand Up @@ -394,19 +422,22 @@ mod test {
let sheet = gc.sheet(sheet_id);

assert_eq!(
sheet.cell_value((0, 0).into()),
sheet.cell_value((0, 1).into()),
Some(CellValue::Number(1.into()))
);
assert_eq!(
sheet.cell_value((2, 9).into()),
sheet.cell_value((2, 10).into()),
Some(CellValue::Number(12.into()))
);
assert_eq!(sheet.cell_value((0, 5).into()), None);
assert_eq!(sheet.cell_value((0, 6).into()), None);
assert_eq!(
sheet.cell_value((3, 1).into()),
Some(CellValue::Number(3.into()))
sheet.cell_value((3, 2).into()),
Some(CellValue::Code(CodeCellValue {
language: CodeCellLanguage::Formula,
code: "C1:C5".into()
}))
);
assert_eq!(sheet.cell_value((3, 0).into()), None);
assert_eq!(sheet.cell_value((3, 1).into()), None);
}

#[test]
Expand Down
Loading

0 comments on commit 08a90d3

Please sign in to comment.