81463223d5
## Summary Addresses #16965 This PR adds support for **opening and saving** files with legacy encodings (non-UTF-8). Previously, Zed failed to open files encoded in Shift-JIS, EUC-JP, Big5, etc., displaying a "Could not open file" error screen. This PR implements automatic encoding detection upon opening and ensures the original encoding is preserved when saving. ## Implementation Details 1. **Worktree (Loading)**: * Updated `load_file` to use `chardetng` for automatic encoding detection. * Files are decoded to UTF-8 internal strings for editing, while preserving the detected `Encoding` metadata. 2. **Language / Buffer**: * Added an `encoding` field to the `Buffer` struct to store the detected encoding. 3. **Worktree (Saving)**: * Updated `write_file` to accept the stored encoding. * **Performance Optimization**: * **UTF-8 Path**: Uses the existing optimized `fs.save` (streaming chunks directly from Rope), ensuring no performance regression for the vast majority of files. * **Legacy Encoding Path**: Implemented a fallback that converts the Rope to a contiguous `String/Bytes` in memory, re-encodes it to the target format (e.g., Shift-JIS), and writes it to disk. * *Note*: This fallback involves memory allocation, but it is necessary to support legacy encodings without refactoring the `fs` crate's streaming interfaces. ## Changes - `crates/worktree`: - Add dependencies: `encoding_rs`, `chardetng`. - Update `load_file` to detect encoding and decode content. - Update `write_file` to handle re-encoding on save. - `crates/language`: Add `encoding` field and accessors to `Buffer`. - `crates/project`: Pass encoding information between Worktree and Buffer. - `crates/vim`: Update `:w` command to use the new `write_file` signature. ## Verification I validated this manually using a Rust script to generate test files with various encodings. **Results:** * ✅ **Success (Opened & Saved correctly):** * **Japanese:** `Shift-JIS` (CP932), `EUC-JP`, `ISO-2022-JP` * **Chinese:** `Big5` (Traditional), `GBK/GB2312` (Simplified) * **Western/Unicode:** `Windows-1252` (CP1252), `UTF-16LE`, `UTF-16BE` * ⚠️ **limitations (Detection accuracy):** * Some specific encodings like `KOI8-R` or generic `Latin1` (ISO-8859-1) may partially display replacement characters (`?`) depending on the file content length. This is a known limitation of the heuristic detection library (`chardetng`) rather than the saving logic. Release Notes: - Added support for opening and saving files with legacy encodings (Shift-JIS, Big5, etc.) --------- Co-authored-by: CrazyboyQCD <53971641+CrazyboyQCD@users.noreply.github.com> Co-authored-by: Conrad Irwin <conrad.irwin@gmail.com>
123 lines
3.4 KiB
TOML
123 lines
3.4 KiB
TOML
[package]
|
|
name = "project"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
publish.workspace = true
|
|
license = "GPL-3.0-or-later"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[lib]
|
|
path = "src/project.rs"
|
|
doctest = false
|
|
|
|
[features]
|
|
test-support = [
|
|
"buffer_diff/test-support",
|
|
"client/test-support",
|
|
"language/test-support",
|
|
"settings/test-support",
|
|
"snippet_provider/test-support",
|
|
"text/test-support",
|
|
"prettier/test-support",
|
|
"worktree/test-support",
|
|
"gpui/test-support",
|
|
"dap/test-support",
|
|
"dap_adapters/test-support",
|
|
]
|
|
|
|
[dependencies]
|
|
aho-corasick.workspace = true
|
|
anyhow.workspace = true
|
|
askpass.workspace = true
|
|
async-trait.workspace = true
|
|
base64.workspace = true
|
|
buffer_diff.workspace = true
|
|
circular-buffer.workspace = true
|
|
client.workspace = true
|
|
clock.workspace = true
|
|
collections.workspace = true
|
|
context_server.workspace = true
|
|
dap.workspace = true
|
|
encoding_rs.workspace = true
|
|
extension.workspace = true
|
|
fancy-regex.workspace = true
|
|
fs.workspace = true
|
|
futures.workspace = true
|
|
fuzzy.workspace = true
|
|
git.workspace = true
|
|
git_hosting_providers.workspace = true
|
|
globset.workspace = true
|
|
gpui.workspace = true
|
|
http_client.workspace = true
|
|
image.workspace = true
|
|
itertools.workspace = true
|
|
indexmap.workspace = true
|
|
language.workspace = true
|
|
log.workspace = true
|
|
lsp.workspace = true
|
|
markdown.workspace = true
|
|
node_runtime.workspace = true
|
|
parking_lot.workspace = true
|
|
paths.workspace = true
|
|
postage.workspace = true
|
|
prettier.workspace = true
|
|
rand.workspace = true
|
|
regex.workspace = true
|
|
remote.workspace = true
|
|
rpc.workspace = true
|
|
schemars.workspace = true
|
|
semver.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
settings.workspace = true
|
|
sha2.workspace = true
|
|
shellexpand.workspace = true
|
|
smallvec.workspace = true
|
|
smol.workspace = true
|
|
snippet.workspace = true
|
|
snippet_provider.workspace = true
|
|
sum_tree.workspace = true
|
|
task.workspace = true
|
|
tempfile.workspace = true
|
|
terminal.workspace = true
|
|
text.workspace = true
|
|
toml.workspace = true
|
|
url.workspace = true
|
|
util.workspace = true
|
|
watch.workspace = true
|
|
wax.workspace = true
|
|
which.workspace = true
|
|
worktree.workspace = true
|
|
zeroize.workspace = true
|
|
zlog.workspace = true
|
|
ztracing.workspace = true
|
|
tracing.workspace = true
|
|
|
|
[dev-dependencies]
|
|
client = { workspace = true, features = ["test-support"] }
|
|
db = { workspace = true, features = ["test-support"] }
|
|
collections = { workspace = true, features = ["test-support"] }
|
|
context_server = { workspace = true, features = ["test-support"] }
|
|
buffer_diff = { workspace = true, features = ["test-support"] }
|
|
dap = { workspace = true, features = ["test-support"] }
|
|
dap_adapters = { workspace = true, features = ["test-support"] }
|
|
fs = { workspace = true, features = ["test-support"] }
|
|
git2.workspace = true
|
|
gpui = { workspace = true, features = ["test-support"] }
|
|
language = { workspace = true, features = ["test-support"] }
|
|
lsp = { workspace = true, features = ["test-support"] }
|
|
prettier = { workspace = true, features = ["test-support"] }
|
|
pretty_assertions.workspace = true
|
|
release_channel.workspace = true
|
|
rpc = { workspace = true, features = ["test-support"] }
|
|
settings = { workspace = true, features = ["test-support"] }
|
|
snippet_provider = { workspace = true, features = ["test-support"] }
|
|
unindent.workspace = true
|
|
util = { workspace = true, features = ["test-support"] }
|
|
worktree = { workspace = true, features = ["test-support"] }
|
|
|
|
[package.metadata.cargo-machete]
|
|
ignored = ["tracing"]
|