diff --git a/pretix_zugferd/invoice.py b/pretix_zugferd/invoice.py
index 7be5fae..a1f86ae 100644
--- a/pretix_zugferd/invoice.py
+++ b/pretix_zugferd/invoice.py
@@ -267,6 +267,7 @@ def _zugferd_generate_document(self, invoice):
         doc.header.notes.add(note)
 
         if invoice.internal_reference:
+            doc.trade.agreement.buyer_reference = invoice.internal_reference
             note = IncludedNote()
             note.content.add(
                 pgettext("invoice", "Customer reference: {reference}").format(
@@ -490,5 +491,6 @@ class Modern1ZugferdXRechnungInvoiceRenderer(ZugferdMixin, Modern1Renderer):
 
     def _zugferd_generate_document(self, invoice):
         doc = super()._zugferd_generate_document(invoice)
-        doc.trade.agreement.buyer_reference = invoice.custom_field or "unknown"
+        if self.event.settings.invoice_address_custom_field:
+            doc.trade.agreement.buyer_reference = invoice.custom_field or "unknown"
         return doc
diff --git a/tests/results/cancellation.xml b/tests/results/cancellation.xml
index 32230dd..65ca795 100644
--- a/tests/results/cancellation.xml
+++ b/tests/results/cancellation.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>
diff --git a/tests/results/default.xml b/tests/results/default.xml
index 06296b1..87ae6a3 100644
--- a/tests/results/default.xml
+++ b/tests/results/default.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>
diff --git a/tests/results/delivery_date_off.xml b/tests/results/delivery_date_off.xml
index f789143..12f13da 100644
--- a/tests/results/delivery_date_off.xml
+++ b/tests/results/delivery_date_off.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>
diff --git a/tests/results/paid.xml b/tests/results/paid.xml
index 6a819df..3e8d0af 100644
--- a/tests/results/paid.xml
+++ b/tests/results/paid.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>
diff --git a/tests/results/tax_code_exempt.xml b/tests/results/tax_code_exempt.xml
index eb90962..3bf14e8 100644
--- a/tests/results/tax_code_exempt.xml
+++ b/tests/results/tax_code_exempt.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>
diff --git a/tests/results/tax_code_guess_AU.xml b/tests/results/tax_code_guess_AU.xml
index cefab83..1f8cbd2 100644
--- a/tests/results/tax_code_guess_AU.xml
+++ b/tests/results/tax_code_guess_AU.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>
diff --git a/tests/results/tax_code_reverse_charge.xml b/tests/results/tax_code_reverse_charge.xml
index 4fdc9cc..fcc7352 100644
--- a/tests/results/tax_code_reverse_charge.xml
+++ b/tests/results/tax_code_reverse_charge.xml
@@ -73,6 +73,7 @@ Attendee: Peter</ram:Description>
             </ram:SpecifiedLineTradeSettlement>
         </ram:IncludedSupplyChainTradeLineItem>
         <ram:ApplicableHeaderTradeAgreement>
+            <ram:BuyerReference>PO-12345</ram:BuyerReference>
             <ram:SellerTradeParty>
                 <ram:Name>Musterfirma</ram:Name>
                 <ram:PostalTradeAddress>