Skip to content

Commit 37a1544

Browse files
MarshalXMarshalX
authored
Add error handling; fix performance; fix cid decoding (#4)
1 parent a99bc9b commit 37a1544

File tree

2 files changed

+130
-160
lines changed

2 files changed

+130
-160
lines changed

‎Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "libipld"
3-
version = "1.1.0"
3+
version = "1.1.1"
44
edition = "2021"
55
license = "MIT"
66
description = "Python binding to the Rust IPLD library"
@@ -13,7 +13,7 @@ name = "libipld"
1313
crate-type = ["cdylib"]
1414

1515
[dependencies]
16-
pyo3 = { version = "0.19", features = ["generate-import-lib", "anyhow"] }
16+
pyo3 = { version = "0.20", features = ["generate-import-lib", "anyhow"] }
1717
python3-dll-a = "0.2.7"
1818
anyhow = "1.0.75"
1919
futures = "0.3"

‎src/lib.rs

Lines changed: 128 additions & 158 deletions
Original file line numberDiff line numberDiff line change
@@ -1,217 +1,187 @@
1-
use std::borrow::Cow;
2-
use std::collections::{HashMap};
31
use std::io::{BufReader, Cursor, Read, Seek};
4-
use pyo3::prelude::*;
5-
use pyo3::conversion::ToPyObject;
6-
use pyo3::{PyObject, Python};
7-
use pyo3::types::{PyBytes};
2+
3+
use ::libipld::{cid::Cid, Ipld};
4+
use ::libipld::cbor::{cbor::MajorKind, DagCborCodec, decode};
5+
use ::libipld::prelude::Codec;
86
use anyhow::Result;
9-
use iroh_car::{CarHeader, CarReader};
107
use futures::{executor, stream::StreamExt};
11-
use ::libipld::cbor::cbor::MajorKind;
12-
use ::libipld::cbor::decode;
13-
use ::libipld::{cid::Cid, Ipld};
14-
15-
16-
#[derive(Clone, PartialEq)]
17-
pub enum HashMapItem {
18-
Null,
19-
Bool(bool),
20-
Integer(i128),
21-
Float(f64),
22-
String(String),
23-
List(Vec<HashMapItem>),
24-
Map(HashMap<String, HashMapItem>),
25-
Bytes(Cow<'static, [u8]>),
26-
}
27-
28-
impl HashMapItem {
29-
fn value(&self) -> PyObject {
30-
Python::with_gil(|py| match self {
31-
Self::Null => py.None(),
32-
Self::Bool(b) => b.to_object(py),
33-
Self::String(s) => s.to_object(py),
34-
Self::Integer(i) => i.to_object(py),
35-
Self::Float(f) => f.to_object(py),
36-
Self::List(l) => l.to_object(py),
37-
Self::Map(m) => m.to_object(py),
38-
Self::Bytes(b) => b.to_object(py),
39-
})
8+
use iroh_car::{CarHeader, CarReader, Error};
9+
use pyo3::{PyObject, Python};
10+
use pyo3::conversion::ToPyObject;
11+
use pyo3::prelude::*;
12+
use pyo3::types::{PyBytes, PyDict, PyList};
13+
14+
fn ipld_to_pyobject(py: Python<'_>, ipld: &Ipld) -> PyObject {
15+
match ipld {
16+
Ipld::Null => py.None(),
17+
Ipld::Bool(b) => b.to_object(py),
18+
Ipld::Integer(i) => i.to_object(py),
19+
Ipld::Float(f) => f.to_object(py),
20+
Ipld::String(s) => s.to_object(py),
21+
Ipld::Bytes(b) => PyBytes::new(py, b).into(),
22+
Ipld::Link(cid) => cid.to_string().to_object(py),
23+
Ipld::List(l) => {
24+
let list_obj = PyList::empty(py);
25+
l.iter().for_each(|item| {
26+
let item_obj = ipld_to_pyobject(py, item);
27+
list_obj.append(item_obj).unwrap();
28+
});
29+
list_obj.into()
30+
}
31+
Ipld::Map(m) => {
32+
let dict_obj = PyDict::new(py);
33+
m.iter().for_each(|(key, value)| {
34+
let key_obj = key.to_object(py);
35+
let value_obj = ipld_to_pyobject(py, value);
36+
dict_obj.set_item(key_obj, value_obj).unwrap();
37+
});
38+
dict_obj.into()
39+
}
4040
}
4141
}
4242

43-
impl ToPyObject for HashMapItem {
44-
fn to_object(&self, _: Python<'_>) -> PyObject {
45-
self.value().into()
46-
}
47-
}
43+
fn car_header_to_pydict<'py>(py: Python<'py>, header: &CarHeader) -> &'py PyDict {
44+
let dict_obj = PyDict::new(py);
4845

49-
impl IntoPy<Py<PyAny>> for HashMapItem {
50-
fn into_py(self, _: Python<'_>) -> Py<PyAny> {
51-
self.value().into()
52-
}
53-
}
46+
dict_obj.set_item("version", header.version()).unwrap();
5447

48+
let roots = PyList::empty(py);
49+
header.roots().iter().for_each(|cid| {
50+
let cid_obj = cid.to_string().to_object(py);
51+
roots.append(cid_obj).unwrap();
52+
});
5553

56-
fn ipld_to_hashmap(x: Ipld) -> HashMapItem {
57-
match x {
58-
Ipld::Null => HashMapItem::Null,
59-
Ipld::Bool(b) => HashMapItem::Bool(b),
60-
Ipld::Integer(i) => HashMapItem::Integer(i),
61-
Ipld::Float(f) => HashMapItem::Float(f),
62-
Ipld::String(s) => HashMapItem::String(s),
63-
Ipld::Bytes(b) => HashMapItem::Bytes(Cow::Owned(b)),
64-
Ipld::List(l) => HashMapItem::List(l.into_iter().map(ipld_to_hashmap).collect()),
65-
Ipld::Map(m) => HashMapItem::Map(
66-
m.into_iter()
67-
.map(|(k, v)| (k, ipld_to_hashmap(v)))
68-
.collect(),
69-
),
70-
Ipld::Link(cid) => HashMapItem::String(cid.to_string()),
71-
}
72-
}
54+
dict_obj.set_item("roots", roots).unwrap();
7355

74-
fn car_header_to_hashmap(header: &CarHeader) -> HashMapItem {
75-
HashMapItem::Map(
76-
vec![
77-
("version".to_string(), HashMapItem::Integer(header.version() as i128)),
78-
(
79-
"roots".to_string(),
80-
HashMapItem::List(
81-
header
82-
.roots()
83-
.iter()
84-
.map(|cid| HashMapItem::String(cid.to_string()))
85-
.collect(),
86-
),
87-
),
88-
]
89-
.into_iter()
90-
.collect(),
91-
)
56+
dict_obj.into()
9257
}
9358

94-
fn _cid_hash_to_hashmap(cid: &Cid) -> HashMapItem {
59+
fn cid_hash_to_pydict<'py>(py: Python<'py>, cid: &Cid) -> &'py PyDict {
9560
let hash = cid.hash();
96-
HashMapItem::Map(
97-
vec![
98-
("code".to_string(), HashMapItem::Integer(hash.code() as i128)),
99-
("size".to_string(), HashMapItem::Integer(hash.size() as i128)),
100-
("digest".to_string(), HashMapItem::Bytes(Cow::Owned(hash.digest().to_vec()))),
101-
]
102-
.into_iter()
103-
.collect(),
104-
)
61+
let dict_obj = PyDict::new(py);
62+
63+
dict_obj.set_item("code", hash.code()).unwrap();
64+
dict_obj.set_item("size", hash.size()).unwrap();
65+
dict_obj.set_item("digest", PyBytes::new(py, &hash.digest())).unwrap();
66+
67+
dict_obj.into()
10568
}
10669

107-
fn cid_to_hashmap(cid: &Cid) -> HashMapItem {
108-
HashMapItem::Map(
109-
vec![
110-
("version".to_string(), HashMapItem::Integer(cid.version() as i128)),
111-
("codec".to_string(), HashMapItem::Integer(cid.codec() as i128)),
112-
("hash".to_string(), _cid_hash_to_hashmap(cid)),
113-
]
114-
.into_iter()
115-
.collect(),
116-
)
70+
fn cid_to_pydict<'py>(py: Python<'py>, cid: &Cid) -> &'py PyDict {
71+
let dict_obj = PyDict::new(py);
72+
73+
dict_obj.set_item("version", cid.version() as u64).unwrap();
74+
dict_obj.set_item("codec", cid.codec()).unwrap();
75+
dict_obj.set_item("hash", cid_hash_to_pydict(py, cid)).unwrap();
76+
77+
dict_obj.into()
11778
}
11879

119-
fn parse_dag_cbor_object<R: Read + Seek>(mut reader: &mut BufReader<R>) -> Result<Ipld> {
120-
let major = decode::read_major(&mut reader)?;
80+
fn parse_dag_cbor_object<R: Read + Seek>(r: &mut R) -> Result<Ipld> {
81+
let major = decode::read_major(r)?;
12182
Ok(match major.kind() {
12283
MajorKind::UnsignedInt | MajorKind::NegativeInt => Ipld::Integer(major.info() as i128),
123-
MajorKind::ByteString => Ipld::Bytes(decode::read_bytes(&mut reader, major.info() as u64)?),
124-
MajorKind::TextString => Ipld::String(decode::read_str(&mut reader, major.info() as u64)?),
125-
MajorKind::Array => Ipld::List(decode::read_list(&mut reader, major.info() as u64)?),
126-
MajorKind::Map => Ipld::Map(decode::read_map(&mut reader, major.info() as u64)?),
84+
MajorKind::ByteString => Ipld::Bytes(decode::read_bytes(r, major.info() as u64)?),
85+
MajorKind::TextString => Ipld::String(decode::read_str(r, major.info() as u64)?),
86+
MajorKind::Array => Ipld::List(decode::read_list(r, major.info() as u64)?),
87+
MajorKind::Map => Ipld::Map(decode::read_map(r, major.info() as u64)?),
12788
MajorKind::Tag => {
12889
if major.info() != 42 {
12990
return Err(anyhow::anyhow!("non-42 tags are not supported"));
13091
}
13192

132-
parse_dag_cbor_object(reader)?
93+
Ipld::Link(decode::read_link(r)?)
13394
}
13495
MajorKind::Other => Ipld::Null,
13596
})
13697
}
13798

13899
#[pyfunction]
139-
fn decode_dag_cbor_multi(data: Vec<u8>) -> PyResult<Vec<HashMapItem>> {
100+
fn decode_dag_cbor_multi(py: Python, data: &[u8]) -> PyResult<Vec<PyObject>> {
140101
let mut reader = BufReader::new(Cursor::new(data));
141-
142102
let mut parts = Vec::new();
103+
143104
loop {
144-
let cbor = parse_dag_cbor_object(&mut reader);
145-
if let Ok(cbor) = cbor {
146-
parts.push(_ipld_to_python(cbor));
105+
let ipld = parse_dag_cbor_object(&mut reader);
106+
if let Ok(cbor) = ipld {
107+
parts.push(ipld_to_pyobject(py, &cbor));
147108
} else {
148109
break;
149110
}
150111
}
112+
151113
Ok(parts)
152114
}
153115

154-
fn _decode_dag_cbor(data: Vec<u8>) -> Result<Ipld> {
155-
let mut reader = BufReader::new(Cursor::new(data));
156-
parse_dag_cbor_object(&mut reader)
157-
}
116+
#[pyfunction]
117+
fn decode_car<'py>(py: Python<'py>, data: &[u8]) -> PyResult<(&'py PyDict, &'py PyDict)> {
118+
let car_response = executor::block_on(CarReader::new(data));
119+
if let Err(e) = car_response {
120+
return Err(get_err("Failed to decode CAR", e.to_string()));
121+
}
158122

159-
fn _ipld_to_python(ipld: Ipld) -> HashMapItem {
160-
ipld_to_hashmap(ipld.clone())
161-
}
123+
let car = car_response.unwrap();
162124

163-
#[pyfunction]
164-
fn decode_car(data: Vec<u8>) -> (HashMapItem, HashMap<String, HashMapItem>) {
165-
let car = executor::block_on(CarReader::new(data.as_slice())).unwrap();
166-
let header = car_header_to_hashmap(car.header());
167-
let blocks = executor::block_on(car
168-
.stream()
169-
.filter_map(|block| async {
170-
if let Ok((cid, bytes)) = block {
171-
let mut reader = BufReader::new(Cursor::new(bytes));
172-
173-
let ipld = parse_dag_cbor_object(&mut reader);
174-
if let Ok(ipld) = ipld {
175-
Some((cid.to_string(), ipld))
176-
} else {
177-
None
178-
}
179-
} else {
180-
None
181-
}
182-
})
183-
.collect::<HashMap<String, Ipld>>());
125+
let header = car_header_to_pydict(py, car.header());
126+
let parsed_blocks = PyDict::new(py);
184127

185-
let mut decoded_blocks = HashMap::new();
186-
for (cid, ipld) in &blocks {
187-
decoded_blocks.insert(cid.to_string(), _ipld_to_python(ipld.clone()));
188-
}
128+
let blocks: Vec<Result<(Cid, Vec<u8>), Error>> = executor::block_on(car.stream().collect());
129+
blocks.into_iter().for_each(|block| {
130+
if let Ok((cid, bytes)) = block {
131+
let ipld = DagCborCodec.decode(&bytes);
132+
if let Ok(ipld) = ipld {
133+
let key = cid.to_string().to_object(py);
134+
let value = ipld_to_pyobject(py, &ipld);
135+
parsed_blocks.set_item(key, value).unwrap();
136+
}
137+
}
138+
});
189139

190-
(header, decoded_blocks)
140+
Ok((header, parsed_blocks))
191141
}
192142

193143
#[pyfunction]
194-
fn decode_dag_cbor(data: Vec<u8>) -> PyResult<HashMapItem> {
195-
Ok(_ipld_to_python(_decode_dag_cbor(data)?))
144+
fn decode_dag_cbor(py: Python, data: &[u8]) -> PyResult<PyObject> {
145+
let ipld = DagCborCodec.decode(data);
146+
if let Ok(ipld) = ipld {
147+
Ok(ipld_to_pyobject(py, &ipld))
148+
} else {
149+
Err(get_err("Failed to decode DAG-CBOR", ipld.unwrap_err().to_string()))
150+
}
196151
}
197152

198153
#[pyfunction]
199-
fn decode_cid(data: String) -> PyResult<HashMapItem> {
200-
let cid = Cid::try_from(data.as_str()).unwrap();
201-
Ok(cid_to_hashmap(&cid))
154+
fn decode_cid(py: Python, data: String) -> PyResult<&PyDict> {
155+
let cid = Cid::try_from(data.as_str());
156+
if let Ok(cid) = cid {
157+
Ok(cid_to_pydict(py, &cid))
158+
} else {
159+
Err(get_err("Failed to decode CID", cid.unwrap_err().to_string()))
160+
}
202161
}
203162

204163
#[pyfunction]
205-
fn decode_multibase(py: Python, data: String) -> (char, PyObject) {
206-
let (base, data) = multibase::decode(data).unwrap();
207-
(base.code(), PyBytes::new(py, &data).into())
164+
fn decode_multibase(py: Python, data: String) -> PyResult<(char, PyObject)> {
165+
let base = multibase::decode(data);
166+
if let Ok((base, data)) = base {
167+
Ok((base.code(), PyBytes::new(py, &data).into()))
168+
} else {
169+
Err(get_err("Failed to decode multibase", base.unwrap_err().to_string()))
170+
}
208171
}
209172

210173
#[pyfunction]
211-
fn encode_multibase(code: char, data: Vec<u8>) -> String {
212-
let base = multibase::Base::from_code(code).unwrap();
213-
let encoded = multibase::encode(base, data);
214-
encoded
174+
fn encode_multibase(code: char, data: &[u8]) -> PyResult<String> {
175+
let base = multibase::Base::from_code(code);
176+
if let Ok(base) = base {
177+
Ok(multibase::encode(base, data))
178+
} else {
179+
Err(get_err("Failed to encode multibase", base.unwrap_err().to_string()))
180+
}
181+
}
182+
183+
fn get_err(msg: &str, err: String) -> PyErr {
184+
PyErr::new::<pyo3::exceptions::PyValueError, _>(format!("{}. {}", msg, err))
215185
}
216186

217187
#[pymodule]

0 commit comments

Comments
 (0)