diff --git a/kwant/qsymm.py b/kwant/qsymm.py
index bb752e19314a18cbc7ec53e53eb163df5bf17f80..728dd2b738373eff85099e073e75b5fb5bb9c802 100644
--- a/kwant/qsymm.py
+++ b/kwant/qsymm.py
@@ -323,8 +323,8 @@ def model_to_builder(model, norbs, lat_vecs, atom_coords, *, coeffs=None):
 
     # Keep track of the hoppings and onsites by storing those
     # which have already been set.
-    hopping_dict = defaultdict(dict)
-    onsites_dict = defaultdict(dict)
+    hopping_dict = defaultdict(lambda: 0)
+    onsites_dict = defaultdict(lambda: 0)
 
     # Iterate over all terms in the model.
     for key, hop_mat in model.items():