add unconditional manual token counting for completions

Author: alexander-alderman-webb

sentry_sdk/integrations/openai.py

@@ -640,6 +640,13 @@ def new_iterator() -> "Iterator[ChatCompletionChunk]":
                     set_data_normalized(
                         span, SPANDATA.GEN_AI_RESPONSE_TEXT, all_responses
                     )
+                _calculate_token_usage(
+                    messages,
+                    response,
+                    span,
+                    all_responses,
+                    integration.count_tokens,
+                )
 
         if finish_span:
             span.__exit__(None, None, None)
@@ -675,6 +682,13 @@ async def new_iterator_async() -> "AsyncIterator[ChatCompletionChunk]":
                     set_data_normalized(
                         span, SPANDATA.GEN_AI_RESPONSE_TEXT, all_responses
                     )
+                _calculate_token_usage(
+                    messages,
+                    response,
+                    span,
+                    all_responses,
+                    integration.count_tokens,
+                )
 
         if finish_span:
             span.__exit__(None, None, None)