Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[UI] Support exporting pod stdout and result #399

Merged
merged 2 commits into from
Jul 28, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions api/src/resolver_export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,9 @@ async function exportJSON(_, { repoId }, { userId }) {
},
},
});
// now export repo to a file
if (!repo) throw Error("Repo not exists.");
const filename = `${
repo.name || "Untitled"
}-${new Date().toISOString()}.json`;
const aws_url = await uploadToS3WithExpiration(
filename,
JSON.stringify({ name: repo.name, version: "v0.0.1", pods: repo.pods })
);
return aws_url;
// return the JSON string
return JSON.stringify({ name: repo.name, format: "codepod", version: "v0.0.1", pods: repo.pods });
}

interface Pod {
Expand Down
2 changes: 1 addition & 1 deletion runtime/src/zmq-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ export function constructExecuteRequest({ code, msg_id, cp = {} }) {
cp,
// FIXME if this is true, no result is returned!
silent: false,
store_history: false,
store_history: true,
// XXX this does not seem to be used
user_expressions: {
x: "3+4",
Expand Down
2 changes: 2 additions & 0 deletions ui/src/components/Canvas.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,8 @@ function CanvasImpl() {
cellList = JSON.parse(String(fileContent)).cells.map((cell) => ({
cellType: cell.cell_type,
cellSource: cell.source.join(""),
cellOutputs: cell.outputs || [],
execution_count: cell.execution_count || 0,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should display the execution count in the output area of a Code pod. This PR already gets packed, let's implement it in another PR.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

execution_count is a property of both a code cell and its output (if the output type is execute_result) #411

}));
importScopeName = fileName.substring(0, fileName.length - 6);
break;
Expand Down
48 changes: 40 additions & 8 deletions ui/src/components/Sidebar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -850,6 +850,9 @@ function ExportFile() {
function ExportJSON() {
// an export component
let { id: repoId } = useParams();
const store = useContext(RepoContext);
if (!store) throw new Error("Missing BearContext.Provider in the tree");
const repoName = useStore(store, (state) => state.repoName);
// the useMutation for exportJSON
const [exportJSON, { data, loading, error }] = useMutation(
gql`
Expand All @@ -860,7 +863,17 @@ function ExportJSON() {
);
useEffect(() => {
if (data?.exportJSON) {
download(data.exportJSON);
let element = document.createElement("a");
element.setAttribute(
"href",
"data:text/plain;charset=utf-8," + encodeURIComponent(data.exportJSON)
);
element.setAttribute("download", `${repoName || ""}.json`);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's add date to the exported filename and keep consistent with the .ipynb export.

const filename = `${
repoName || "Untitled"
}-${new Date().toISOString()}.ipynb`;

Copy link
Collaborator

@lihebi lihebi Jul 28, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nevermind, I made this minor change. Merging.


element.style.display = "none";
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
}
}, [data]);
return (
Expand All @@ -870,10 +883,10 @@ function ExportJSON() {
size="small"
color="secondary"
onClick={() => {
// call export graphQL api to get the AWS S3 url
// call export graphQL api to get JSON string
exportJSON({ variables: { repoId } });
}}
disabled={true}
disabled={false}
>
Raw JSON
</Button>
Expand All @@ -899,9 +912,10 @@ function ExportJupyterNB() {
// Hard-code Jupyter cell format. Reference, https://nbformat.readthedocs.io/en/latest/format_description.html
let jupyterCellList: {
cell_type: string;
execution_count: number;
execution_count?: number;
metadata: object;
source: string[];
outputs?: object[];
}[] = [];

// Queue to sort the pods geographically
Expand Down Expand Up @@ -941,19 +955,36 @@ function ExportJupyterNB() {
if (pod.type == "SCOPE") {
q.push([pod, geoScore.substring(0, 2) + "0" + geoScore.substring(2)]);
} else if (pod.type == "CODE") {
let podOutput: any[] = [];
if (pod.stdout) {
podOutput.push({
output_type: "stream",
name: "stdout",
text: pod.stdout.split(/\r?\n/).map((line) => line + "\n"),
});
}
if (pod.result) {
podOutput.push({
output_type: "display_data",
data: {
"text/plain": (pod.result.text || "")
.split(/\r?\n/)
.map((line) => line + "\n") || [""],
"image/png": pod.result.image,
},
});
}
jupyterCellList.push({
cell_type: "code",
// hard-code execution_count
execution_count: 1,
execution_count: pod.result?.count || 0,
// TODO: expand other Codepod related-metadata fields, or run a real-time search in database when importing.
metadata: { id: pod.id, geoScore: Number(geoScore) },
source: [pod.content || ""],
outputs: podOutput,
});
} else if (pod.type == "RICH") {
jupyterCellList.push({
cell_type: "markdown",
// hard-code execution_count
execution_count: 1,
// TODO: expand other Codepod related-metadata fields, or run a real-time search in database when importing.
metadata: { id: pod.id, geoScore: Number(geoScore) },
source: [pod.richContent || ""],
Expand Down Expand Up @@ -1029,6 +1060,7 @@ function ExportJupyterNB() {
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
setLoading(false);
};

return (
Expand Down
27 changes: 26 additions & 1 deletion ui/src/lib/store/canvasSlice.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,7 @@ export const createCanvasSlice: StateCreator<MyState, [], [], CanvasSlice> = (
dirty: true,
pending: true,
});
let maxLineLength = 0;
if (cellList.length > 0) {
for (let i = 0; i < cellList.length; i++) {
const cell = cellList[i];
Expand All @@ -529,8 +530,29 @@ export const createCanvasSlice: StateCreator<MyState, [], [], CanvasSlice> = (
newPos
);
let podContent = cell.cellType == "code" ? cell.cellSource : "";

maxLineLength = Math.max(
maxLineLength,
Math.max(...podContent.split(/\r?\n/).map((line) => line.length))
);

let podRichContent = cell.cellType == "markdown" ? cell.cellSource : "";

let podResult = { count: cell.execution_count, text: "", image: "" };
let podStdOut = "";

for (const cellOutput of cell.cellOutputs) {
if (
cellOutput["output_type"] === "stream" &&
cellOutput["name"] === "stdout"
) {
podStdOut = cellOutput["text"].join("");
}
if (cellOutput["output_type"] === "display_data") {
podResult.text = cellOutput["data"]["text/plain"].join("");
podResult.image = cellOutput["data"]["image/png"];
}
}
// move the created node to scope and configure the necessary node attributes
const posInsideScope = getNodePositionInsideScope(
node,
Expand Down Expand Up @@ -572,6 +594,8 @@ export const createCanvasSlice: StateCreator<MyState, [], [], CanvasSlice> = (
height: node.height!,
content: podContent,
richContent: podRichContent,
stdout: podStdOut === "" ? undefined : podStdOut,
result: podResult.text === "" ? undefined : podResult,
// For my local update, set dirty to true to push to DB.
dirty: true,
pending: true,
Expand All @@ -587,7 +611,8 @@ export const createCanvasSlice: StateCreator<MyState, [], [], CanvasSlice> = (
}
get().adjustLevel();
get().buildNode2Children();
// Set initial width as about 30 characters.
// Set initial width as a scale of max line length.
//get().setNodeCharWidth(scopeNode.id, Math.ceil(maxLineLength * 0.8));
get().setNodeCharWidth(scopeNode.id, 30);
get().updateView();
},
Expand Down