diff --git a/README.md b/README.md index bddf44b..3d90e8c 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,39 @@ See `tools/modly-cli/SKILL.md` for the agent workflow and output contract. Join the [Discord server](https://discord.gg/BvjDCvS3yr) to stay up to date with the latest news, report bugs, and share feedback. +Follow Modly and its development on X: + +- [Modly on X](https://x.com/modly3d) +- [Lightning Pixel on X](https://x.com/lightningpiixel) + +--- + +## Sponsors + +

+ Thanks to our early sponsors for believing in Modly and helping make local AI 3D generation more accessible. +

+ +

+ + DrHepa +
+ DrHepa + +    + + benjapenjamin +
+ benjapenjamin + +    + + iammojogo-sudo +
+ iammojogo-sudo + +

+ --- ## License diff --git a/api/main.py b/api/main.py index e3c5b75..db4006e 100644 --- a/api/main.py +++ b/api/main.py @@ -31,7 +31,7 @@ def filter(self, record): app = FastAPI( title="Modly API", - version="0.3.5", + version="0.3.6", lifespan=lifespan, ) diff --git a/api/routers/optimize.py b/api/routers/optimize.py index 282e723..e78ddd5 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -5,6 +5,343 @@ import tempfile import uuid +try: + import pymeshlab as _pymeshlab + _PYMESHLAB_AVAILABLE = True +except ImportError: + _pymeshlab = None + _PYMESHLAB_AVAILABLE = False + +import numpy as np +import trimesh +import trimesh.visual +from fastapi import APIRouter, HTTPException, UploadFile, File +from fastapi.responses import FileResponse, Response +from pathlib import Path +from urllib.parse import quote +from pydantic import BaseModel + +from services.generator_registry import WORKSPACE_DIR + +router = APIRouter(tags=["optimize"]) + + +class OptimizeRequest(BaseModel): + path: str # format: "{collection}/{filename}" + target_faces: int + + +class SmoothRequest(BaseModel): + path: str # format: "{collection}/{filename}" + iterations: int + + +class TransformRequest(BaseModel): + path: str # format: "{collection}/{filename}" + matrix: list[list[float]] # row-major 4x4 world transform + + +def _require_pymeshlab(): + if not _PYMESHLAB_AVAILABLE: + raise HTTPException(503, "pymeshlab is unavailable on this system (DLL blocked by Windows Application Control policy)") + + +def _resolve_input_path(raw_path: str) -> Path: + candidate = Path(raw_path) + if candidate.is_absolute(): + resolved = candidate.resolve() + if not resolved.exists(): + raise HTTPException(404, f"File not found: {raw_path}") + return resolved + + resolved = (WORKSPACE_DIR / raw_path).resolve() + if not str(resolved).startswith(str(WORKSPACE_DIR.resolve())): + raise HTTPException(400, "Invalid path") + if not resolved.exists(): + raise HTTPException(404, f"File not found: {raw_path}") + return resolved + + +@router.post("/mesh") +def optimize_mesh(body: OptimizeRequest): + _require_pymeshlab() + target_faces = max(100, min(500_000, body.target_faces)) + + input_path = _resolve_input_path(body.path) + + tmp_dir = tempfile.mkdtemp() + try: + result = _decimate(str(input_path), target_faces, tmp_dir) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + stem = input_path.stem + output_name = f"{stem}_opt{target_faces}.glb" + output_dir = input_path.parent if str(input_path).startswith(str(WORKSPACE_DIR.resolve())) else WORKSPACE_DIR / "Workflows" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / output_name + result.export(str(output_path)) + + face_count = len(result.faces) + rel = output_path.relative_to(WORKSPACE_DIR).as_posix() + return {"url": f"/workspace/{rel}", "face_count": face_count} + + +def _has_texture(geom: trimesh.Trimesh) -> bool: + if not isinstance(geom.visual, trimesh.visual.TextureVisuals): + return False + mat = geom.visual.material + if mat is None: + return False + # Simple material (SimpleMaterial / Material) + if getattr(mat, "image", None) is not None: + return True + # PBR material (from Trellis2 SLaT texturing and GLB imports) + if getattr(mat, "baseColorTexture", None) is not None: + return True + return False + + +def _get_texture_image(geom: trimesh.Trimesh): + """Return the base color texture image regardless of material type.""" + mat = geom.visual.material + img = getattr(mat, "image", None) + if img is not None: + return img + return getattr(mat, "baseColorTexture", None) + + +def _decimate(input_path: str, target_faces: int, tmp_dir: str) -> trimesh.Trimesh: + loaded = trimesh.load(input_path) + if isinstance(loaded, trimesh.Scene): + geoms = list(loaded.geometry.values()) + geom = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] + else: + geom = loaded + + ms = _pymeshlab.MeshSet() + + if _has_texture(geom): + # ── Textured path: OBJ intermediate to preserve UV coordinates ────── + obj_in = os.path.join(tmp_dir, "input.obj") + mtl_in = os.path.join(tmp_dir, "input.mtl") + tex_in = os.path.join(tmp_dir, "texture.png") + obj_out = os.path.join(tmp_dir, "output.obj") + + # Save texture image under a known filename (handles PBR and simple materials) + _get_texture_image(geom).save(tex_in) + + # Export OBJ (trimesh writes UV coords + MTL) + geom.export(obj_in) + + # Patch MTL so any map_Kd points to our known texture filename + if os.path.exists(mtl_in): + mtl = open(mtl_in).read() + mtl = re.sub(r"map_Kd\s+\S+", "map_Kd texture.png", mtl) + open(mtl_in, "w").write(mtl) + + ms.load_new_mesh(obj_in) + ms.meshing_decimation_quadric_edge_collapse( + targetfacenum=target_faces, + preservetexcoord=True, # ← keeps UV coordinates intact + preservenormal=True, + preservetopology=True, + autoclean=True, + ) + ms.save_current_mesh(obj_out) + + # Patch output MTL too, so trimesh can find the texture on load + mtl_out = obj_out.replace(".obj", ".mtl") + if os.path.exists(mtl_out): + mtl = open(mtl_out).read() + mtl = re.sub(r"map_Kd\s+\S+", "map_Kd texture.png", mtl) + open(mtl_out, "w").write(mtl) + + return trimesh.load(obj_out) + + else: + # ── Geometry-only path: PLY (fast, no texture to worry about) ──────── + ply_in = os.path.join(tmp_dir, "input.ply") + ply_out = os.path.join(tmp_dir, "output.ply") + + geom.export(ply_in) + ms.load_new_mesh(ply_in) + ms.meshing_decimation_quadric_edge_collapse( + targetfacenum=target_faces, + preservenormal=True, + preservetopology=True, + autoclean=True, + ) + ms.save_current_mesh(ply_out) + return trimesh.load(ply_out, force="mesh") + + +@router.post("/smooth") +def smooth_mesh(body: SmoothRequest): + _require_pymeshlab() + iterations = max(1, min(20, body.iterations)) + + input_path = _resolve_input_path(body.path) + + tmp_dir = tempfile.mkdtemp() + try: + result = _smooth(str(input_path), iterations, tmp_dir) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + stem = input_path.stem + output_name = f"{stem}_smooth{iterations}.glb" + output_dir = input_path.parent if str(input_path).startswith(str(WORKSPACE_DIR.resolve())) else WORKSPACE_DIR / "Workflows" + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / output_name + result.export(str(output_path)) + + rel = output_path.relative_to(WORKSPACE_DIR).as_posix() + return {"url": f"/workspace/{rel}"} + + +def _smooth(input_path: str, iterations: int, tmp_dir: str) -> trimesh.Trimesh: + loaded = trimesh.load(input_path) + if isinstance(loaded, trimesh.Scene): + geoms = list(loaded.geometry.values()) + geom = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] + else: + geom = loaded + + ms = _pymeshlab.MeshSet() + + if _has_texture(geom): + obj_in = os.path.join(tmp_dir, "input.obj") + mtl_in = os.path.join(tmp_dir, "input.mtl") + tex_in = os.path.join(tmp_dir, "texture.png") + obj_out = os.path.join(tmp_dir, "output.obj") + + _get_texture_image(geom).save(tex_in) + geom.export(obj_in) + + if os.path.exists(mtl_in): + mtl = open(mtl_in).read() + mtl = re.sub(r"map_Kd\s+\S+", "map_Kd texture.png", mtl) + open(mtl_in, "w").write(mtl) + + ms.load_new_mesh(obj_in) + ms.apply_coord_laplacian_smoothing(stepsmoothnum=iterations) + ms.save_current_mesh(obj_out) + + mtl_out = obj_out.replace(".obj", ".mtl") + if os.path.exists(mtl_out): + mtl = open(mtl_out).read() + mtl = re.sub(r"map_Kd\s+\S+", "map_Kd texture.png", mtl) + open(mtl_out, "w").write(mtl) + + return trimesh.load(obj_out) + + else: + ply_in = os.path.join(tmp_dir, "input.ply") + ply_out = os.path.join(tmp_dir, "output.ply") + + geom.export(ply_in) + ms.load_new_mesh(ply_in) + ms.apply_coord_laplacian_smoothing(stepsmoothnum=iterations) + ms.save_current_mesh(ply_out) + return trimesh.load(ply_out, force="mesh") + + +@router.post("/transform") +def transform_mesh(body: TransformRequest): + input_path = (WORKSPACE_DIR / body.path).resolve() + if not str(input_path).startswith(str(WORKSPACE_DIR.resolve())): + raise HTTPException(400, "Invalid path") + if not input_path.exists(): + raise HTTPException(404, f"File not found: {body.path}") + + matrix = np.asarray(body.matrix, dtype=float) + if matrix.shape != (4, 4): + raise HTTPException(400, "matrix must be a 4x4 array") + if not np.all(np.isfinite(matrix)): + raise HTTPException(400, "matrix contains non-finite values") + + loaded = trimesh.load(str(input_path)) + loaded.apply_transform(matrix) + + stem = input_path.stem + output_name = f"{stem}_xf.glb" + output_path = input_path.parent / output_name + loaded.export(str(output_path)) + + collection_name = body.path.split("/")[0] + return {"url": f"/workspace/{collection_name}/{output_name}"} + + +class ImportByPathRequest(BaseModel): + path: str # absolute path on disk + + +@router.post("/import-by-path") +async def import_mesh_by_path(body: ImportByPathRequest): + file_path = Path(body.path) + if not file_path.is_file(): + raise HTTPException(400, "File not found") + + ext = file_path.suffix.lstrip(".").lower() + if ext not in ("glb", "obj", "stl", "ply"): + raise HTTPException(400, f"Unsupported format: {ext}") + + if ext == "glb": + # Serve the original file directly — no copy + return {"url": f"/optimize/serve-file?path={quote(str(file_path))}"} + + # Non-GLB: convert to GLB in a temp directory (not the workspace) + tmp_dir = tempfile.mkdtemp(prefix="modly_import_") + output_path = os.path.join(tmp_dir, "mesh.glb") + loaded = trimesh.load(str(file_path)) + loaded.export(output_path) + return {"url": f"/optimize/serve-file?path={quote(output_path)}"} + + +@router.get("/serve-file") +def serve_file(path: str): + file_path = Path(path) + if not file_path.is_file(): + raise HTTPException(404, "File not found") + if file_path.suffix.lower() != ".glb": + raise HTTPException(400, "Only GLB files can be served") + return FileResponse(str(file_path), media_type="model/gltf-binary") + + +@router.get("/export") +def export_mesh(path: str, format: str): + if format not in ("obj", "stl", "ply"): + raise HTTPException(400, "Supported formats: obj, stl, ply") + + input_path = (WORKSPACE_DIR / path).resolve() + if not str(input_path).startswith(str(WORKSPACE_DIR.resolve())): + raise HTTPException(400, "Invalid path") + if not input_path.exists(): + raise HTTPException(404, f"File not found: {path}") + + loaded = trimesh.load(str(input_path)) + if isinstance(loaded, trimesh.Scene): + geoms = list(loaded.geometry.values()) + mesh = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] + else: + mesh = loaded + + data = mesh.export(file_type=format) + stem = input_path.stem + mime = "text/plain" if format == "obj" else "application/octet-stream" + # trimesh exports ply as bytes even in text mode — octet-stream is fine for all binary formats + return Response( + content=data, + media_type=mime, + headers={"Content-Disposition": f'attachment; filename="{stem}.{format}"'}, + ) +import os +import re +import shutil +import tempfile +import uuid + try: import pymeshlab as _pymeshlab _PYMESHLAB_AVAILABLE = True diff --git a/package.json b/package.json index a3fc6db..7928e32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "modly", - "version": "0.3.5", + "version": "0.3.6", "description": "Local AI-powered 3D mesh generation from images", "main": "./out/main/index.js", "author": "Modly", diff --git a/src/areas/generate/components/Viewer3D.tsx b/src/areas/generate/components/Viewer3D.tsx index 4224fe0..11e500b 100644 --- a/src/areas/generate/components/Viewer3D.tsx +++ b/src/areas/generate/components/Viewer3D.tsx @@ -13,6 +13,7 @@ THREE.BufferGeometry.prototype.disposeBoundsTree = disposeBoundsTree as any THREE.Mesh.prototype.raycast = acceleratedRaycast import SplatViewer, { type SplatViewerHandle } from './SplatViewer' import { useGeneration } from '@shared/hooks/useGeneration' +import { useApi } from '@shared/hooks/useApi' import { useAppStore } from '@shared/stores/appStore' import { ViewerToolbar, type ViewMode } from './ViewerToolbar' import type { LightSettings } from '@shared/stores/appStore' @@ -137,6 +138,7 @@ interface MeshModelProps { selected: boolean onStats: (stats: { vertices: number; triangles: number }) => void onSelect: () => void + onObject: (obj: THREE.Object3D | null) => void } function MeshModel({ url, jobId, viewMode, selected, onStats, onSelect }: MeshModelProps): JSX.Element { @@ -208,17 +210,23 @@ function SceneMeshModel({ } }, [scene]) - // Centre the mesh on the grid + // Centre the mesh on the grid. + // Only runs on first load / model change (autoCenter) or an explicit Reset + // (resetToken) — never on plain re-renders, so a live gizmo transform or a + // baked "Apply" pose is not silently overwritten. useEffect(() => { - // Reset before computing — useGLTF caches the scene with its already-modified position, - // which would skew the setFromObject (world space) on a second mount. - scene.position.set(0, 0, 0) - const box = new THREE.Box3().setFromObject(scene) - const center = new THREE.Vector3() - box.getCenter(center) - scene.position.set(-center.x, -box.min.y, -center.z) - - // Compute stats + if (autoCenter) { + // Clear any live gizmo transform before measuring. + scene.position.set(0, 0, 0) + scene.rotation.set(0, 0, 0) + scene.scale.set(1, 1, 1) + const box = new THREE.Box3().setFromObject(scene) + const center = new THREE.Vector3() + box.getCenter(center) + scene.position.set(-center.x, -box.min.y, -center.z) + } + + // Compute stats (independent of centering) let vertices = 0 let triangles = 0 scene.traverse((child) => { @@ -231,7 +239,7 @@ function SceneMeshModel({ }) const roundedTriangles = Math.round(triangles) onStats({ vertices: Math.round(vertices), triangles: roundedTriangles }) - }, [scene]) + }, [scene, autoCenter, resetToken]) // Thumbnail capture (kept for future use) useEffect(() => { @@ -387,6 +395,9 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l const setStoreMeshStats = useAppStore((s) => s.setMeshStats) const meshStats = useAppStore((s) => s.meshStats) const setCurrentJob = useAppStore((s) => s.setCurrentJob) + const updateCurrentJob = useAppStore((s) => s.updateCurrentJob) + const pushMeshUrl = useAppStore((s) => s.pushMeshUrl) + const { transformMesh } = useApi() const [viewMode, setViewMode] = useState('solid') const [autoRotate, setAutoRotate] = useState(false) @@ -415,6 +426,7 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l useEffect(() => { setSelected(false) setViewMode('solid') + setGizmoMode(null) setStoreMeshStats(null) }, [modelUrl]) @@ -425,11 +437,24 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l // Delete key removes the model from the scene useEffect(() => { const handler = (e: KeyboardEvent) => { - if (e.key !== 'Delete') return - if (document.activeElement instanceof HTMLInputElement) return - if (!selected) return - setCurrentJob(null) - setSelected(false) + const el = document.activeElement + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) return + + if (e.key === 'Delete') { + if (!selected) return + setCurrentJob(null) + setSelected(false) + return + } + if (e.key === 'Escape') { + setGizmoMode(null) + return + } + const key = e.key.toLowerCase() + if (key === 'w' || key === 'e' || key === 'r') { + if (!selected) return + setGizmoMode(key === 'w' ? 'translate' : key === 'e' ? 'rotate' : 'scale') + } } window.addEventListener('keydown', handler) return () => window.removeEventListener('keydown', handler) @@ -446,6 +471,56 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l link.click() } + const handleSelect = () => { + setSelected(true) + setGizmoMode((m) => m ?? 'translate') + } + + const handleGizmoMode = (mode: GizmoMode) => { + setSelected(true) + setGizmoMode((m) => (m === mode ? null : mode)) + } + + const handleResetTransform = () => { + setGizmoMode(null) + const original = currentJob?.originalOutputUrl + if (original && currentJob?.outputUrl !== original) { + // Was baked via Apply — reload the original (auto-centred on load). + updateCurrentJob({ outputUrl: original }) + pushMeshUrl(original) + } else { + // Live transform only — re-centre the current scene in place. + setResetToken((t) => t + 1) + } + } + + const handleApplyTransform = async () => { + if (!meshObject || !currentJob?.outputUrl) return + const url = currentJob.outputUrl + const path = url.replace('/workspace/', '') + + meshObject.updateWorldMatrix(true, false) + const e = meshObject.matrixWorld.elements + // THREE stores column-major; emit a row-major 4x4 for the backend. + const matrix = [ + [e[0], e[4], e[8], e[12]], + [e[1], e[5], e[9], e[13]], + [e[2], e[6], e[10], e[14]], + [e[3], e[7], e[11], e[15]], + ] + + setApplying(true) + try { + const result = await transformMesh(path, matrix) + appliedUrls.current.add(result.url) + setGizmoMode(null) + updateCurrentJob({ outputUrl: result.url }) + pushMeshUrl(result.url) + } finally { + setApplying(false) + } + } + return ( }> @@ -511,6 +586,10 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l ) : null} + {selected && gizmoMode && meshObject && ( + + )} + setAutoRotate((v) => !v)} onScreenshot={handleScreenshot} @@ -556,7 +637,7 @@ export default function Viewer3D({ lightSettings = DEFAULT_LIGHT_SETTINGS }: { l

{selected - ? <>Click mesh to select • Delete to remove + ? <>W/E/R move/rotate/scale • Esc exit • Delete remove : 'Drag to rotate \u2022 Scroll to zoom' }

diff --git a/src/areas/generate/components/ViewerToolbar.tsx b/src/areas/generate/components/ViewerToolbar.tsx index c83d4f3..4b13e44 100644 --- a/src/areas/generate/components/ViewerToolbar.tsx +++ b/src/areas/generate/components/ViewerToolbar.tsx @@ -1,9 +1,11 @@ -import type { ViewMode } from '../models' -export type { ViewMode } +import type { ViewMode, GizmoMode } from '../models' +export type { ViewMode, GizmoMode } interface ViewerToolbarProps { viewMode: ViewMode autoRotate: boolean + gizmoMode: GizmoMode | null + gizmoBusy: boolean onViewMode: (mode: ViewMode) => void onAutoRotate: () => void onScreenshot: () => void @@ -67,9 +69,46 @@ const MODES: { mode: ViewMode; icon: React.ReactNode; label: string }[] = [ }, ] + +const GIZMOS: { mode: GizmoMode; key: string; label: string; icon: React.ReactNode }[] = [ + { + mode: 'translate', + key: 'W', + label: 'Move (W)', + icon: ( + + + + ), + }, + { + mode: 'rotate', + key: 'E', + label: 'Rotate (E)', + icon: ( + + + + + ), + }, + { + mode: 'scale', + key: 'R', + label: 'Scale (R)', + icon: ( + + + + ), + }, +] + export function ViewerToolbar({ viewMode, autoRotate, + gizmoMode, + gizmoBusy, onViewMode, onAutoRotate, onScreenshot, @@ -86,6 +125,19 @@ export function ViewerToolbar({ > {icon} +))} + +
+ + {GIZMOS.map(({ mode, icon, label }) => ( + onGizmoMode(mode)} + > + {icon} + ))} {showViewModes &&
} @@ -120,15 +172,18 @@ interface ToolbarButtonProps { label: string onClick: () => void children: React.ReactNode + disabled?: boolean } -function ToolbarButton({ active, label, onClick, children }: ToolbarButtonProps): JSX.Element { +function ToolbarButton({ active, label, onClick, children, disabled = false }: ToolbarButtonProps): JSX.Element { return (